n	1	// Copyright 2009 The Go Authors. All rights reserved.	n	1	// Copyright 2009 The Go Authors. All rights reserved.
	2	// Use of this source code is governed by a BSD-style		2	// Use of this source code is governed by a BSD-style
	3	// license that can be found in the LICENSE file.		3	// license that can be found in the LICENSE file.
	4			4
	5	#include "zasm_GOOS_GOARCH.h"		5	#include "zasm_GOOS_GOARCH.h"
	6	#include "funcdata.h"		6	#include "funcdata.h"
	7	#include "textflag.h"		7	#include "textflag.h"
	8			8
n	9	TEXT runtime·rt0_go(SB),NOSPLIT,$0	n	9	TEXT runtime·rt0_go(SB), NOSPLIT, $0
	10	// copy arguments forward on an even stack		10	// copy arguments forward on an even stack
n	11	MOVQ DI, AX // argc	n	11	MOVQ DI, AX // argc
	12	MOVQ SI, BX // argv		12	MOVQ SI, BX // argv
	13	SUBQ $(4*8+7), SP // 2args 2auto		13	SUBQ $(4*8+7), SP // 2args 2auto
	14	ANDQ $~15, SP		14	ANDQ $~15, SP
	15	MOVQ AX, 16(SP)		15	MOVQ AX, 16(SP)
	16	MOVQ BX, 24(SP)		16	MOVQ BX, 24(SP)
	17			17
	18	// create istack out of the given (operating system) stack.		18	// create istack out of the given (operating system) stack.
	19	// _cgo_init may update stackguard.		19	// _cgo_init may update stackguard.
n	20	MOVQ $runtime·g0(SB), DI	n	20	MOVQ $runtime·g0(SB), DI
	21	LEAQ (-64*1024+104)(SP), BX		21	LEAQ (-64*1024+104)(SP), BX
	22	MOVQ BX, g_stackguard0(DI)		22	MOVQ BX, g_stackguard0(DI)
	23	MOVQ BX, g_stackguard1(DI)		23	MOVQ BX, g_stackguard1(DI)
	24	MOVQ BX, (g_stack+stack_lo)(DI)		24	MOVQ BX, (g_stack+stack_lo)(DI)
	25	MOVQ SP, (g_stack+stack_hi)(DI)		25	MOVQ SP, (g_stack+stack_hi)(DI)
	26			26
	27	// find out information about the processor we're on		27	// find out information about the processor we're on
n	28	MOVQ $0, AX	n	28	MOVQ $0, AX
	29	CPUID		29	CPUID
n	30	CMPQ AX, $0	n	30	CMPQ AX, $0
	31	JE nocpuinfo		31	JE nocpuinfo
	32	MOVQ $1, AX		32	MOVQ $1, AX
	33	CPUID		33	CPUID
n	34	MOVL CX, runtime·cpuid_ecx(SB)	n	34	MOVL CX, runtime·cpuid_ecx(SB)
	35	MOVL DX, runtime·cpuid_edx(SB)		35	MOVL DX, runtime·cpuid_edx(SB)
				36
	36	nocpuinfo:		37	nocpuinfo:
	37			38
	38	// if there is an _cgo_init, call it.		39	// if there is an _cgo_init, call it.
n	39	MOVQ _cgo_init(SB), AX	n	40	MOVQ _cgo_init(SB), AX
	40	TESTQ AX, AX		41	TESTQ AX, AX
	41	JZ needtls		42	JZ needtls
				43
	42	// g0 already in DI		44	// g0 already in DI
n	43	MOVQ DI, CX // Win64 uses CX for first parameter	n	45	MOVQ DI, CX // Win64 uses CX for first parameter
	44	MOVQ $setg_gcc<>(SB), SI		46	MOVQ $setg_gcc<>(SB), SI
	45	CALL AX		47	CALL AX
	46			48
	47	// update stackguard after _cgo_init		49	// update stackguard after _cgo_init
n	48	MOVQ $runtime·g0(SB), CX	n	50	MOVQ $runtime·g0(SB), CX
	49	MOVQ (g_stack+stack_lo)(CX), AX		51	MOVQ (g_stack+stack_lo)(CX), AX
	50	ADDQ $const_StackGuard, AX		52	ADDQ $const_StackGuard, AX
	51	MOVQ AX, g_stackguard0(CX)		53	MOVQ AX, g_stackguard0(CX)
	52	MOVQ AX, g_stackguard1(CX)		54	MOVQ AX, g_stackguard1(CX)
	53			55
n	54	CMPL runtime·iswindows(SB), $0	n	56	CMPL runtime·iswindows(SB), $0
	55	JEQ ok		57	JEQ ok
				58
	56	needtls:		59	needtls:
	57	// skip TLS setup on Plan 9		60	// skip TLS setup on Plan 9
n	58	CMPL runtime·isplan9(SB), $1	n	61	CMPL runtime·isplan9(SB), $1
	59	JEQ ok		62	JEQ ok
				63
	60	// skip TLS setup on Solaris		64	// skip TLS setup on Solaris
n	61	CMPL runtime·issolaris(SB), $1	n	65	CMPL runtime·issolaris(SB), $1
	62	JEQ ok		66	JEQ ok
	63			67
n	64	LEAQ runtime·tls0(SB), DI	n	68	LEAQ runtime·tls0(SB), DI
	65	CALL runtime·settls(SB)		69	CALL runtime·settls(SB)
	66			70
	67	// store through it, to make sure it works		71	// store through it, to make sure it works
	68	get_tls(BX)		72	get_tls(BX)
n	69	MOVQ $0x123, g(BX)	n	73	MOVQ $0x123, g(BX)
	70	MOVQ runtime·tls0(SB), AX		74	MOVQ runtime·tls0(SB), AX
	71	CMPQ AX, $0x123		75	CMPQ AX, $0x123
	72	JEQ 2(PC)		76	JEQ 2(PC)
	73	MOVL AX, 0 // abort		77	MOVL AX, 0 // abort
				78
	74	ok:		79	ok:
	75	// set the per-goroutine and per-mach "registers"		80	// set the per-goroutine and per-mach "registers"
	76	get_tls(BX)		81	get_tls(BX)
n	77	LEAQ runtime·g0(SB), CX	n	82	LEAQ runtime·g0(SB), CX
	78	MOVQ CX, g(BX)		83	MOVQ CX, g(BX)
	79	LEAQ runtime·m0(SB), AX		84	LEAQ runtime·m0(SB), AX
	80			85
	81	// save m->g0 = g0		86	// save m->g0 = g0
n	82	MOVQ CX, m_g0(AX)	n	87	MOVQ CX, m_g0(AX)
				88
	83	// save m0 to g0->m		89	// save m0 to g0->m
n	84	MOVQ AX, g_m(CX)	n	90	MOVQ AX, g_m(CX)
	85			91
n	86	CLD // convention is D is always left cleared	n	92	CLD // convention is D is always left cleared
	87	CALL runtime·check(SB)		93	CALL runtime·check(SB)
	88			94
n	89	MOVL 16(SP), AX // copy argc	n	95	MOVL 16(SP), AX // copy argc
	90	MOVL AX, 0(SP)		96	MOVL AX, 0(SP)
	91	MOVQ 24(SP), AX // copy argv		97	MOVQ 24(SP), AX // copy argv
	92	MOVQ AX, 8(SP)		98	MOVQ AX, 8(SP)
	93	CALL runtime·args(SB)		99	CALL runtime·args(SB)
	94	CALL runtime·osinit(SB)		100	CALL runtime·osinit(SB)
	95	CALL runtime·schedinit(SB)		101	CALL runtime·schedinit(SB)
	96			102
	97	// create a new goroutine to start program		103	// create a new goroutine to start program
n	98	MOVQ $runtime·main·f(SB), BP // entry	n	104	MOVQ $runtime·main·f(SB), BP // entry
	99	PUSHQ BP		105	PUSHQ BP
	100	PUSHQ $0 // arg size		106	PUSHQ $0 // arg size
	101	CALL runtime·newproc(SB)		107	CALL runtime·newproc(SB)
	102	POPQ AX		108	POPQ AX
	103	POPQ AX		109	POPQ AX
	104			110
	105	// start this M		111	// start this M
n	106	CALL runtime·mstart(SB)	n	112	CALL runtime·mstart(SB)
	107			113
n	108	MOVL $0xf1, 0xf1 // crash	n	114	MOVL $0xf1, 0xf1 // crash
	109	RET		115	RET
	110			116
n	111	DATA runtime·main·f+0(SB)/8,$runtime·main(SB)	n	117	DATA runtime·main·f+0(SB)/8, $runtime·main(SB)
	112	GLOBL runtime·main·f(SB),RODATA,$8		118	GLOBL runtime·main·f(SB), RODATA, $8
	113			119
n	114	TEXT runtime·breakpoint(SB),NOSPLIT,$0-0	n	120	TEXT runtime·breakpoint(SB), NOSPLIT, $0-0
	115	BYTE $0xcc		121	BYTE $0xcc
	116	RET		122	RET
	117			123
n	118	TEXT runtime·asminit(SB),NOSPLIT,$0-0	n	124	TEXT runtime·asminit(SB), NOSPLIT, $0-0
	119	// No per-thread init.		125	// No per-thread init.
	120	RET		126	RET
	121			127
	122	/*		128	/*
	123	* go-routine		129	* go-routine
	124	*/		130	*/
n	125		n
	126	// void gosave(Gobuf*)		131	// void gosave(Gobuf*)
	127	// save state in Gobuf; setjmp		132	// save state in Gobuf; setjmp
	128	TEXT runtime·gosave(SB), NOSPLIT, $0-8		133	TEXT runtime·gosave(SB), NOSPLIT, $0-8
n	129	MOVQ buf+0(FP), AX // gobuf	n	134	MOVQ buf+0(FP), AX // gobuf
	130	LEAQ buf+0(FP), BX // caller's SP		135	LEAQ buf+0(FP), BX // caller's SP
	131	MOVQ BX, gobuf_sp(AX)		136	MOVQ BX, gobuf_sp(AX)
	132	MOVQ 0(SP), BX // caller's PC		137	MOVQ 0(SP), BX // caller's PC
	133	MOVQ BX, gobuf_pc(AX)		138	MOVQ BX, gobuf_pc(AX)
	134	MOVQ $0, gobuf_ret(AX)		139	MOVQ $0, gobuf_ret(AX)
	135	MOVQ $0, gobuf_ctxt(AX)		140	MOVQ $0, gobuf_ctxt(AX)
	136	get_tls(CX)		141	get_tls(CX)
n	137	MOVQ g(CX), BX	n	142	MOVQ g(CX), BX
	138	MOVQ BX, gobuf_g(AX)		143	MOVQ BX, gobuf_g(AX)
	139	RET		144	RET
	140			145
	141	// void gogo(Gobuf*)		146	// void gogo(Gobuf*)
	142	// restore state from Gobuf; longjmp		147	// restore state from Gobuf; longjmp
	143	TEXT runtime·gogo(SB), NOSPLIT, $0-8		148	TEXT runtime·gogo(SB), NOSPLIT, $0-8
n	144	MOVQ buf+0(FP), BX // gobuf	n	149	MOVQ buf+0(FP), BX // gobuf
	145	MOVQ gobuf_g(BX), DX		150	MOVQ gobuf_g(BX), DX
	146	MOVQ 0(DX), CX // make sure g != nil		151	MOVQ 0(DX), CX // make sure g != nil
	147	get_tls(CX)		152	get_tls(CX)
n	148	MOVQ DX, g(CX)	n	153	MOVQ DX, g(CX)
	149	MOVQ gobuf_sp(BX), SP // restore SP		154	MOVQ gobuf_sp(BX), SP // restore SP
	150	MOVQ gobuf_ret(BX), AX		155	MOVQ gobuf_ret(BX), AX
	151	MOVQ gobuf_ctxt(BX), DX		156	MOVQ gobuf_ctxt(BX), DX
	152	MOVQ $0, gobuf_sp(BX) // clear to help garbage collector		157	MOVQ $0, gobuf_sp(BX) // clear to help garbage collector
	153	MOVQ $0, gobuf_ret(BX)		158	MOVQ $0, gobuf_ret(BX)
	154	MOVQ $0, gobuf_ctxt(BX)		159	MOVQ $0, gobuf_ctxt(BX)
	155	MOVQ gobuf_pc(BX), BX		160	MOVQ gobuf_pc(BX), BX
	156	JMP BX		161	JMP BX
	157			162
	158	// func mcall(fn func(*g))		163	// func mcall(fn func(*g))
	159	// Switch to m->g0's stack, call fn(g).		164	// Switch to m->g0's stack, call fn(g).
	160	// Fn must never return. It should gogo(&g->sched)		165	// Fn must never return. It should gogo(&g->sched)
	161	// to keep running g.		166	// to keep running g.
	162	TEXT runtime·mcall(SB), NOSPLIT, $0-8		167	TEXT runtime·mcall(SB), NOSPLIT, $0-8
n	163	MOVQ fn+0(FP), DI	n	168	MOVQ fn+0(FP), DI
	164			169
	165	get_tls(CX)		170	get_tls(CX)
n	166	MOVQ g(CX), AX // save state in g->sched	n	171	MOVQ g(CX), AX // save state in g->sched
	167	MOVQ 0(SP), BX // caller's PC		172	MOVQ 0(SP), BX // caller's PC
	168	MOVQ BX, (g_sched+gobuf_pc)(AX)		173	MOVQ BX, (g_sched+gobuf_pc)(AX)
	169	LEAQ fn+0(FP), BX // caller's SP		174	LEAQ fn+0(FP), BX // caller's SP
	170	MOVQ BX, (g_sched+gobuf_sp)(AX)		175	MOVQ BX, (g_sched+gobuf_sp)(AX)
	171	MOVQ AX, (g_sched+gobuf_g)(AX)		176	MOVQ AX, (g_sched+gobuf_g)(AX)
	172			177
	173	// switch to m->g0 & its stack, call fn		178	// switch to m->g0 & its stack, call fn
n	174	MOVQ g(CX), BX	n	179	MOVQ g(CX), BX
	175	MOVQ g_m(BX), BX		180	MOVQ g_m(BX), BX
	176	MOVQ m_g0(BX), SI		181	MOVQ m_g0(BX), SI
	177	CMPQ SI, AX // if g == m->g0 call badmcall		182	CMPQ SI, AX // if g == m->g0 call badmcall
	178	JNE 3(PC)		183	JNE 3(PC)
	179	MOVQ $runtime·badmcall(SB), AX		184	MOVQ $runtime·badmcall(SB), AX
	180	JMP AX		185	JMP AX
	181	MOVQ SI, g(CX) // g = m->g0		186	MOVQ SI, g(CX) // g = m->g0
	182	MOVQ (g_sched+gobuf_sp)(SI), SP // sp = m->g0->sched.sp		187	MOVQ (g_sched+gobuf_sp)(SI), SP // sp = m->g0->sched.sp
	183	PUSHQ AX		188	PUSHQ AX
	184	MOVQ DI, DX		189	MOVQ DI, DX
	185	MOVQ 0(DI), DI		190	MOVQ 0(DI), DI
	186	CALL DI		191	CALL DI
	187	POPQ AX		192	POPQ AX
	188	MOVQ $runtime·badmcall2(SB), AX		193	MOVQ $runtime·badmcall2(SB), AX
	189	JMP AX		194	JMP AX
	190	RET		195	RET
	191			196
	192	// switchtoM is a dummy routine that onM leaves at the bottom		197	// switchtoM is a dummy routine that onM leaves at the bottom
	193	// of the G stack. We need to distinguish the routine that		198	// of the G stack. We need to distinguish the routine that
	194	// lives at the bottom of the G stack from the one that lives		199	// lives at the bottom of the G stack from the one that lives
	195	// at the top of the M stack because the one at the top of		200	// at the top of the M stack because the one at the top of
	196	// the M stack terminates the stack walk (see topofstack()).		201	// the M stack terminates the stack walk (see topofstack()).
	197	TEXT runtime·switchtoM(SB), NOSPLIT, $0-0		202	TEXT runtime·switchtoM(SB), NOSPLIT, $0-0
	198	RET		203	RET
	199			204
	200	// func onM_signalok(fn func())		205	// func onM_signalok(fn func())
	201	TEXT runtime·onM_signalok(SB), NOSPLIT, $0-8		206	TEXT runtime·onM_signalok(SB), NOSPLIT, $0-8
	202	get_tls(CX)		207	get_tls(CX)
n	203	MOVQ g(CX), AX // AX = g	n	208	MOVQ g(CX), AX // AX = g
	204	MOVQ g_m(AX), BX // BX = m		209	MOVQ g_m(AX), BX // BX = m
	205	MOVQ m_gsignal(BX), DX // DX = gsignal		210	MOVQ m_gsignal(BX), DX // DX = gsignal
	206	CMPQ AX, DX		211	CMPQ AX, DX
	207	JEQ ongsignal		212	JEQ ongsignal
	208	JMP runtime·onM(SB)		213	JMP runtime·onM(SB)
	209			214
	210	ongsignal:		215	ongsignal:
n	211	MOVQ fn+0(FP), DI // DI = fn	n	216	MOVQ fn+0(FP), DI // DI = fn
	212	MOVQ DI, DX		217	MOVQ DI, DX
	213	MOVQ 0(DI), DI		218	MOVQ 0(DI), DI
	214	CALL DI		219	CALL DI
	215	RET		220	RET
	216			221
	217	// func onM(fn func())		222	// func onM(fn func())
	218	TEXT runtime·onM(SB), NOSPLIT, $0-8		223	TEXT runtime·onM(SB), NOSPLIT, $0-8
n	219	MOVQ fn+0(FP), DI // DI = fn	n	224	MOVQ fn+0(FP), DI // DI = fn
	220	get_tls(CX)		225	get_tls(CX)
n	221	MOVQ g(CX), AX // AX = g	n	226	MOVQ g(CX), AX // AX = g
	222	MOVQ g_m(AX), BX // BX = m		227	MOVQ g_m(AX), BX // BX = m
	223			228
n	224	MOVQ m_g0(BX), DX // DX = g0	n	229	MOVQ m_g0(BX), DX // DX = g0
	225	CMPQ AX, DX		230	CMPQ AX, DX
	226	JEQ onm		231	JEQ onm
	227			232
n	228	MOVQ m_curg(BX), BP	n	233	MOVQ m_curg(BX), BP
	229	CMPQ AX, BP		234	CMPQ AX, BP
	230	JEQ oncurg		235	JEQ oncurg
	231			236
	232	// Not g0, not curg. Must be gsignal, but that's not allowed.		237	// Not g0, not curg. Must be gsignal, but that's not allowed.
	233	// Hide call from linker nosplit analysis.		238	// Hide call from linker nosplit analysis.
n	234	MOVQ $runtime·badonm(SB), AX	n	239	MOVQ $runtime·badonm(SB), AX
	235	CALL AX		240	CALL AX
	236			241
	237	oncurg:		242	oncurg:
	238	// save our state in g->sched. Pretend to		243	// save our state in g->sched. Pretend to
	239	// be switchtoM if the G stack is scanned.		244	// be switchtoM if the G stack is scanned.
n	240	MOVQ $runtime·switchtoM(SB), BP	n	245	MOVQ $runtime·switchtoM(SB), BP
	241	MOVQ BP, (g_sched+gobuf_pc)(AX)		246	MOVQ BP, (g_sched+gobuf_pc)(AX)
	242	MOVQ SP, (g_sched+gobuf_sp)(AX)		247	MOVQ SP, (g_sched+gobuf_sp)(AX)
	243	MOVQ AX, (g_sched+gobuf_g)(AX)		248	MOVQ AX, (g_sched+gobuf_g)(AX)
	244			249
	245	// switch to g0		250	// switch to g0
n	246	MOVQ DX, g(CX)	n	251	MOVQ DX, g(CX)
	247	MOVQ (g_sched+gobuf_sp)(DX), BX		252	MOVQ (g_sched+gobuf_sp)(DX), BX
				253
	248	// make it look like mstart called onM on g0, to stop traceback		254	// make it look like mstart called onM on g0, to stop traceback
n	249	SUBQ $8, BX	n	255	SUBQ $8, BX
	250	MOVQ $runtime·mstart(SB), DX		256	MOVQ $runtime·mstart(SB), DX
	251	MOVQ DX, 0(BX)		257	MOVQ DX, 0(BX)
	252	MOVQ BX, SP		258	MOVQ BX, SP
	253			259
	254	// call target function		260	// call target function
n	255	MOVQ DI, DX	n	261	MOVQ DI, DX
	256	MOVQ 0(DI), DI		262	MOVQ 0(DI), DI
	257	CALL DI		263	CALL DI
	258			264
	259	// switch back to g		265	// switch back to g
	260	get_tls(CX)		266	get_tls(CX)
n	261	MOVQ g(CX), AX	n	267	MOVQ g(CX), AX
	262	MOVQ g_m(AX), BX		268	MOVQ g_m(AX), BX
	263	MOVQ m_curg(BX), AX		269	MOVQ m_curg(BX), AX
	264	MOVQ AX, g(CX)		270	MOVQ AX, g(CX)
	265	MOVQ (g_sched+gobuf_sp)(AX), SP		271	MOVQ (g_sched+gobuf_sp)(AX), SP
	266	MOVQ $0, (g_sched+gobuf_sp)(AX)		272	MOVQ $0, (g_sched+gobuf_sp)(AX)
	267	RET		273	RET
	268			274
	269	onm:		275	onm:
	270	// already on m stack, just call directly		276	// already on m stack, just call directly
n	271	MOVQ DI, DX	n	277	MOVQ DI, DX
	272	MOVQ 0(DI), DI		278	MOVQ 0(DI), DI
	273	CALL DI		279	CALL DI
	274	RET		280	RET
	275			281
	276	/*		282	/*
	277	* support for morestack		283	* support for morestack
	278	*/		284	*/
n	279		n
	280	// Called during function prolog when more stack is needed.		285	// Called during function prolog when more stack is needed.
	281	//		286	//
	282	// The traceback routines see morestack on a g0 as being		287	// The traceback routines see morestack on a g0 as being
	283	// the top of a stack (for example, morestack calling newstack		288	// the top of a stack (for example, morestack calling newstack
	284	// calling the scheduler calling newm calling gc), so we must		289	// calling the scheduler calling newm calling gc), so we must
	285	// record an argument size. For that purpose, it has no arguments.		290	// record an argument size. For that purpose, it has no arguments.
n	286	TEXT runtime·morestack(SB),NOSPLIT,$0-0	n	291	TEXT runtime·morestack(SB), NOSPLIT, $0-0
	287	// Cannot grow scheduler stack (m->g0).		292	// Cannot grow scheduler stack (m->g0).
	288	get_tls(CX)		293	get_tls(CX)
n	289	MOVQ g(CX), BX	n	294	MOVQ g(CX), BX
	290	MOVQ g_m(BX), BX		295	MOVQ g_m(BX), BX
	291	MOVQ m_g0(BX), SI		296	MOVQ m_g0(BX), SI
	292	CMPQ g(CX), SI		297	CMPQ g(CX), SI
	293	JNE 2(PC)		298	JNE 2(PC)
	294	INT $3		299	INT $3
	295			300
	296	// Cannot grow signal stack (m->gsignal).		301	// Cannot grow signal stack (m->gsignal).
n	297	MOVQ m_gsignal(BX), SI	n	302	MOVQ m_gsignal(BX), SI
	298	CMPQ g(CX), SI		303	CMPQ g(CX), SI
	299	JNE 2(PC)		304	JNE 2(PC)
	300	INT $3		305	INT $3
	301			306
	302	// Called from f.		307	// Called from f.
	303	// Set m->morebuf to f's caller.		308	// Set m->morebuf to f's caller.
n	304	MOVQ 8(SP), AX // f's caller's PC	n	309	MOVQ 8(SP), AX // f's caller's PC
	305	MOVQ AX, (m_morebuf+gobuf_pc)(BX)		310	MOVQ AX, (m_morebuf+gobuf_pc)(BX)
	306	LEAQ 16(SP), AX // f's caller's SP		311	LEAQ 16(SP), AX // f's caller's SP
	307	MOVQ AX, (m_morebuf+gobuf_sp)(BX)		312	MOVQ AX, (m_morebuf+gobuf_sp)(BX)
	308	get_tls(CX)		313	get_tls(CX)
n	309	MOVQ g(CX), SI	n	314	MOVQ g(CX), SI
	310	MOVQ SI, (m_morebuf+gobuf_g)(BX)		315	MOVQ SI, (m_morebuf+gobuf_g)(BX)
	311			316
	312	// Set g->sched to context in f.		317	// Set g->sched to context in f.
n	313	MOVQ 0(SP), AX // f's PC	n	318	MOVQ 0(SP), AX // f's PC
	314	MOVQ AX, (g_sched+gobuf_pc)(SI)		319	MOVQ AX, (g_sched+gobuf_pc)(SI)
	315	MOVQ SI, (g_sched+gobuf_g)(SI)		320	MOVQ SI, (g_sched+gobuf_g)(SI)
	316	LEAQ 8(SP), AX // f's SP		321	LEAQ 8(SP), AX // f's SP
	317	MOVQ AX, (g_sched+gobuf_sp)(SI)		322	MOVQ AX, (g_sched+gobuf_sp)(SI)
	318	MOVQ DX, (g_sched+gobuf_ctxt)(SI)		323	MOVQ DX, (g_sched+gobuf_ctxt)(SI)
	319			324
	320	// Call newstack on m->g0's stack.		325	// Call newstack on m->g0's stack.
n	321	MOVQ m_g0(BX), BP	n	326	MOVQ m_g0(BX), BP
	322	MOVQ BP, g(CX)		327	MOVQ BP, g(CX)
	323	MOVQ (g_sched+gobuf_sp)(BP), SP		328	MOVQ (g_sched+gobuf_sp)(BP), SP
	324	CALL runtime·newstack(SB)		329	CALL runtime·newstack(SB)
	325	MOVQ $0, 0x1003 // crash if newstack returns		330	MOVQ $0, 0x1003 // crash if newstack returns
	326	RET		331	RET
	327			332
	328	// morestack but not preserving ctxt.		333	// morestack but not preserving ctxt.
n	329	TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0	n	334	TEXT runtime·morestack_noctxt(SB), NOSPLIT, $0
	330	MOVL $0, DX		335	MOVL $0, DX
	331	JMP runtime·morestack(SB)		336	JMP runtime·morestack(SB)
	332			337
	333	// reflectcall: call a function with the given argument list		338	// reflectcall: call a function with the given argument list
	334	// func call(f FuncVal, arg byte, argsize, retoffset uint32).		339	// func call(f FuncVal, arg byte, argsize, retoffset uint32).
	335	// we don't have variable-sized frames, so we use a small number		340	// we don't have variable-sized frames, so we use a small number
	336	// of constant-sized-frame functions to encode a few bits of size in the pc.		341	// of constant-sized-frame functions to encode a few bits of size in the pc.
	337	// Caution: ugly multiline assembly macros in your future!		342	// Caution: ugly multiline assembly macros in your future!
	338			343
n	339	#define DISPATCH(NAME,MAXSIZE) \	n	344	#define DISPATCH(NAME, MAXSIZE) \
	340	CMPQ CX, $MAXSIZE; \		345	CMPQ CX, $MAXSIZE; \
	341	JA 3(PC); \		346	JA 3(PC); \
	342	MOVQ $NAME(SB), AX; \		347	MOVQ $NAME(SB), AX; \
	343	JMP AX		348	JMP AX
				349
	344	// Note: can't just "JMP NAME(SB)" - bad inlining results.		350	// Note: can't just "JMP NAME(SB)" - bad inlining results.
	345			351
	346	TEXT ·reflectcall(SB), NOSPLIT, $0-24		352	TEXT ·reflectcall(SB), NOSPLIT, $0-24
	347	MOVLQZX argsize+16(FP), CX		353	MOVLQZX argsize+16(FP), CX
	348	DISPATCH(runtime·call16, 16)		354	DISPATCH(runtime·call16, 16)
	349	DISPATCH(runtime·call32, 32)		355	DISPATCH(runtime·call32, 32)
	350	DISPATCH(runtime·call64, 64)		356	DISPATCH(runtime·call64, 64)
	351	DISPATCH(runtime·call128, 128)		357	DISPATCH(runtime·call128, 128)
	352	DISPATCH(runtime·call256, 256)		358	DISPATCH(runtime·call256, 256)
	353	DISPATCH(runtime·call512, 512)		359	DISPATCH(runtime·call512, 512)
	354	DISPATCH(runtime·call1024, 1024)		360	DISPATCH(runtime·call1024, 1024)
	355	DISPATCH(runtime·call2048, 2048)		361	DISPATCH(runtime·call2048, 2048)
	356	DISPATCH(runtime·call4096, 4096)		362	DISPATCH(runtime·call4096, 4096)
	357	DISPATCH(runtime·call8192, 8192)		363	DISPATCH(runtime·call8192, 8192)
	358	DISPATCH(runtime·call16384, 16384)		364	DISPATCH(runtime·call16384, 16384)
	359	DISPATCH(runtime·call32768, 32768)		365	DISPATCH(runtime·call32768, 32768)
	360	DISPATCH(runtime·call65536, 65536)		366	DISPATCH(runtime·call65536, 65536)
	361	DISPATCH(runtime·call131072, 131072)		367	DISPATCH(runtime·call131072, 131072)
	362	DISPATCH(runtime·call262144, 262144)		368	DISPATCH(runtime·call262144, 262144)
	363	DISPATCH(runtime·call524288, 524288)		369	DISPATCH(runtime·call524288, 524288)
	364	DISPATCH(runtime·call1048576, 1048576)		370	DISPATCH(runtime·call1048576, 1048576)
	365	DISPATCH(runtime·call2097152, 2097152)		371	DISPATCH(runtime·call2097152, 2097152)
	366	DISPATCH(runtime·call4194304, 4194304)		372	DISPATCH(runtime·call4194304, 4194304)
	367	DISPATCH(runtime·call8388608, 8388608)		373	DISPATCH(runtime·call8388608, 8388608)
	368	DISPATCH(runtime·call16777216, 16777216)		374	DISPATCH(runtime·call16777216, 16777216)
	369	DISPATCH(runtime·call33554432, 33554432)		375	DISPATCH(runtime·call33554432, 33554432)
	370	DISPATCH(runtime·call67108864, 67108864)		376	DISPATCH(runtime·call67108864, 67108864)
	371	DISPATCH(runtime·call134217728, 134217728)		377	DISPATCH(runtime·call134217728, 134217728)
	372	DISPATCH(runtime·call268435456, 268435456)		378	DISPATCH(runtime·call268435456, 268435456)
	373	DISPATCH(runtime·call536870912, 536870912)		379	DISPATCH(runtime·call536870912, 536870912)
	374	DISPATCH(runtime·call1073741824, 1073741824)		380	DISPATCH(runtime·call1073741824, 1073741824)
n	375	MOVQ $runtime·badreflectcall(SB), AX	n	381	MOVQ $runtime·badreflectcall(SB), AX
	376	JMP AX		382	JMP AX
	377			383
n	378	#define CALLFN(NAME,MAXSIZE) \	n	384	#define CALLFN(NAME, MAXSIZE) \
	379	TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \		385	TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \
	380	NO_LOCAL_POINTERS; \		386	NO_LOCAL_POINTERS; \
	381	/* copy arguments to stack */ \		387	/* copy arguments to stack */ \
	382	MOVQ argptr+8(FP), SI; \		388	MOVQ argptr+8(FP), SI; \
	383	MOVLQZX argsize+16(FP), CX; \		389	MOVLQZX argsize+16(FP), CX; \
	384	MOVQ SP, DI; \		390	MOVQ SP, DI; \
	385	REP;MOVSB; \		391	REP;MOVSB; \
	386	/* call function */ \		392	/* call function */ \
	387	MOVQ f+0(FP), DX; \		393	MOVQ f+0(FP), DX; \
	388	PCDATA $PCDATA_StackMapIndex, $0; \		394	PCDATA $PCDATA_StackMapIndex, $0; \
	389	CALL (DX); \		395	CALL (DX); \
	390	/* copy return values back */ \		396	/* copy return values back */ \
	391	MOVQ argptr+8(FP), DI; \		397	MOVQ argptr+8(FP), DI; \
	392	MOVLQZX argsize+16(FP), CX; \		398	MOVLQZX argsize+16(FP), CX; \
	393	MOVLQZX retoffset+20(FP), BX; \		399	MOVLQZX retoffset+20(FP), BX; \
	394	MOVQ SP, SI; \		400	MOVQ SP, SI; \
	395	ADDQ BX, DI; \		401	ADDQ BX, DI; \
	396	ADDQ BX, SI; \		402	ADDQ BX, SI; \
	397	SUBQ BX, CX; \		403	SUBQ BX, CX; \
	398	REP;MOVSB; \		404	REP;MOVSB; \
	399	RET		405	RET
	400			406
	401	CALLFN(·call16, 16)		407	CALLFN(·call16, 16)
	402	CALLFN(·call32, 32)		408	CALLFN(·call32, 32)
	403	CALLFN(·call64, 64)		409	CALLFN(·call64, 64)
	404	CALLFN(·call128, 128)		410	CALLFN(·call128, 128)
	405	CALLFN(·call256, 256)		411	CALLFN(·call256, 256)
	406	CALLFN(·call512, 512)		412	CALLFN(·call512, 512)
	407	CALLFN(·call1024, 1024)		413	CALLFN(·call1024, 1024)
	408	CALLFN(·call2048, 2048)		414	CALLFN(·call2048, 2048)
	409	CALLFN(·call4096, 4096)		415	CALLFN(·call4096, 4096)
	410	CALLFN(·call8192, 8192)		416	CALLFN(·call8192, 8192)
	411	CALLFN(·call16384, 16384)		417	CALLFN(·call16384, 16384)
	412	CALLFN(·call32768, 32768)		418	CALLFN(·call32768, 32768)
	413	CALLFN(·call65536, 65536)		419	CALLFN(·call65536, 65536)
	414	CALLFN(·call131072, 131072)		420	CALLFN(·call131072, 131072)
	415	CALLFN(·call262144, 262144)		421	CALLFN(·call262144, 262144)
	416	CALLFN(·call524288, 524288)		422	CALLFN(·call524288, 524288)
	417	CALLFN(·call1048576, 1048576)		423	CALLFN(·call1048576, 1048576)
	418	CALLFN(·call2097152, 2097152)		424	CALLFN(·call2097152, 2097152)
	419	CALLFN(·call4194304, 4194304)		425	CALLFN(·call4194304, 4194304)
	420	CALLFN(·call8388608, 8388608)		426	CALLFN(·call8388608, 8388608)
	421	CALLFN(·call16777216, 16777216)		427	CALLFN(·call16777216, 16777216)
	422	CALLFN(·call33554432, 33554432)		428	CALLFN(·call33554432, 33554432)
	423	CALLFN(·call67108864, 67108864)		429	CALLFN(·call67108864, 67108864)
	424	CALLFN(·call134217728, 134217728)		430	CALLFN(·call134217728, 134217728)
	425	CALLFN(·call268435456, 268435456)		431	CALLFN(·call268435456, 268435456)
	426	CALLFN(·call536870912, 536870912)		432	CALLFN(·call536870912, 536870912)
	427	CALLFN(·call1073741824, 1073741824)		433	CALLFN(·call1073741824, 1073741824)
	428			434
	429	// bool cas(int32 *val, int32 old, int32 new)		435	// bool cas(int32 *val, int32 old, int32 new)
	430	// Atomically:		436	// Atomically:
	431	// if(*val == old){		437	// if(*val == old){
	432	// *val = new;		438	// *val = new;
	433	// return 1;		439	// return 1;
	434	// } else		440	// } else
	435	// return 0;		441	// return 0;
	436	TEXT runtime·cas(SB), NOSPLIT, $0-17		442	TEXT runtime·cas(SB), NOSPLIT, $0-17
n	437	MOVQ ptr+0(FP), BX	n	443	MOVQ ptr+0(FP), BX
	438	MOVL old+8(FP), AX		444	MOVL old+8(FP), AX
	439	MOVL new+12(FP), CX		445	MOVL new+12(FP), CX
	440	LOCK		446	LOCK
n	441	CMPXCHGL CX, 0(BX)	n	447	CMPXCHGL CX, 0(BX)
	442	JZ 4(PC)		448	JZ 4(PC)
	443	MOVL $0, AX		449	MOVL $0, AX
	444	MOVB AX, ret+16(FP)		450	MOVB AX, ret+16(FP)
	445	RET		451	RET
n	446	MOVL $1, AX	n	452	MOVL $1, AX
	447	MOVB AX, ret+16(FP)		453	MOVB AX, ret+16(FP)
	448	RET		454	RET
	449			455
	450	// bool runtime·cas64(uint64 *val, uint64 old, uint64 new)		456	// bool runtime·cas64(uint64 *val, uint64 old, uint64 new)
	451	// Atomically:		457	// Atomically:
	452	// if(val == old){		458	// if(val == old){
	453	// *val = new;		459	// *val = new;
	454	// return 1;		460	// return 1;
	455	// } else {		461	// } else {
	456	// return 0;		462	// return 0;
	457	// }		463	// }
	458	TEXT runtime·cas64(SB), NOSPLIT, $0-25		464	TEXT runtime·cas64(SB), NOSPLIT, $0-25
n	459	MOVQ ptr+0(FP), BX	n	465	MOVQ ptr+0(FP), BX
	460	MOVQ old+8(FP), AX		466	MOVQ old+8(FP), AX
	461	MOVQ new+16(FP), CX		467	MOVQ new+16(FP), CX
	462	LOCK		468	LOCK
n	463	CMPXCHGQ CX, 0(BX)	n	469	CMPXCHGQ CX, 0(BX)
	464	JNZ cas64_fail		470	JNZ cas64_fail
	465	MOVL $1, AX		471	MOVL $1, AX
	466	MOVB AX, ret+24(FP)		472	MOVB AX, ret+24(FP)
	467	RET		473	RET
n			n	474
	468	cas64_fail:		475	cas64_fail:
n	469	MOVL $0, AX	n	476	MOVL $0, AX
	470	MOVB AX, ret+24(FP)		477	MOVB AX, ret+24(FP)
	471	RET		478	RET
n	472		n	479
	473	TEXT runtime·casuintptr(SB), NOSPLIT, $0-25		480	TEXT runtime·casuintptr(SB), NOSPLIT, $0-25
n	474	JMP runtime·cas64(SB)	n	481	JMP runtime·cas64(SB)
	475			482
	476	TEXT runtime·atomicloaduintptr(SB), NOSPLIT, $0-16		483	TEXT runtime·atomicloaduintptr(SB), NOSPLIT, $0-16
n	477	JMP runtime·atomicload64(SB)	n	484	JMP runtime·atomicload64(SB)
	478			485
	479	TEXT runtime·atomicloaduint(SB), NOSPLIT, $0-16		486	TEXT runtime·atomicloaduint(SB), NOSPLIT, $0-16
n	480	JMP runtime·atomicload64(SB)	n	487	JMP runtime·atomicload64(SB)
	481			488
	482	TEXT runtime·atomicstoreuintptr(SB), NOSPLIT, $0-16		489	TEXT runtime·atomicstoreuintptr(SB), NOSPLIT, $0-16
n	483	JMP runtime·atomicstore64(SB)	n	490	JMP runtime·atomicstore64(SB)
	484			491
	485	// bool casp(void *val, void old, void *new)		492	// bool casp(void *val, void old, void *new)
	486	// Atomically:		493	// Atomically:
	487	// if(*val == old){		494	// if(*val == old){
	488	// *val = new;		495	// *val = new;
	489	// return 1;		496	// return 1;
	490	// } else		497	// } else
	491	// return 0;		498	// return 0;
	492	TEXT runtime·casp(SB), NOSPLIT, $0-25		499	TEXT runtime·casp(SB), NOSPLIT, $0-25
n	493	MOVQ ptr+0(FP), BX	n	500	MOVQ ptr+0(FP), BX
	494	MOVQ old+8(FP), AX		501	MOVQ old+8(FP), AX
	495	MOVQ new+16(FP), CX		502	MOVQ new+16(FP), CX
	496	LOCK		503	LOCK
n	497	CMPXCHGQ CX, 0(BX)	n	504	CMPXCHGQ CX, 0(BX)
	498	JZ 4(PC)		505	JZ 4(PC)
	499	MOVL $0, AX		506	MOVL $0, AX
	500	MOVB AX, ret+24(FP)		507	MOVB AX, ret+24(FP)
	501	RET		508	RET
n	502	MOVL $1, AX	n	509	MOVL $1, AX
	503	MOVB AX, ret+24(FP)		510	MOVB AX, ret+24(FP)
	504	RET		511	RET
	505			512
	506	// uint32 xadd(uint32 volatile *val, int32 delta)		513	// uint32 xadd(uint32 volatile *val, int32 delta)
	507	// Atomically:		514	// Atomically:
	508	// *val += delta;		515	// *val += delta;
	509	// return *val;		516	// return *val;
	510	TEXT runtime·xadd(SB), NOSPLIT, $0-20		517	TEXT runtime·xadd(SB), NOSPLIT, $0-20
n	511	MOVQ ptr+0(FP), BX	n	518	MOVQ ptr+0(FP), BX
	512	MOVL delta+8(FP), AX		519	MOVL delta+8(FP), AX
	513	MOVL AX, CX		520	MOVL AX, CX
	514	LOCK		521	LOCK
n	515	XADDL AX, 0(BX)	n	522	XADDL AX, 0(BX)
	516	ADDL CX, AX		523	ADDL CX, AX
	517	MOVL AX, ret+16(FP)		524	MOVL AX, ret+16(FP)
	518	RET		525	RET
	519			526
	520	TEXT runtime·xadd64(SB), NOSPLIT, $0-24		527	TEXT runtime·xadd64(SB), NOSPLIT, $0-24
n	521	MOVQ ptr+0(FP), BX	n	528	MOVQ ptr+0(FP), BX
	522	MOVQ delta+8(FP), AX		529	MOVQ delta+8(FP), AX
	523	MOVQ AX, CX		530	MOVQ AX, CX
	524	LOCK		531	LOCK
n	525	XADDQ AX, 0(BX)	n	532	XADDQ AX, 0(BX)
	526	ADDQ CX, AX		533	ADDQ CX, AX
	527	MOVQ AX, ret+16(FP)		534	MOVQ AX, ret+16(FP)
	528	RET		535	RET
	529			536
	530	TEXT runtime·xchg(SB), NOSPLIT, $0-20		537	TEXT runtime·xchg(SB), NOSPLIT, $0-20
n	531	MOVQ ptr+0(FP), BX	n	538	MOVQ ptr+0(FP), BX
	532	MOVL new+8(FP), AX		539	MOVL new+8(FP), AX
	533	XCHGL AX, 0(BX)		540	XCHGL AX, 0(BX)
	534	MOVL AX, ret+16(FP)		541	MOVL AX, ret+16(FP)
	535	RET		542	RET
	536			543
	537	TEXT runtime·xchg64(SB), NOSPLIT, $0-24		544	TEXT runtime·xchg64(SB), NOSPLIT, $0-24
n	538	MOVQ ptr+0(FP), BX	n	545	MOVQ ptr+0(FP), BX
	539	MOVQ new+8(FP), AX		546	MOVQ new+8(FP), AX
	540	XCHGQ AX, 0(BX)		547	XCHGQ AX, 0(BX)
	541	MOVQ AX, ret+16(FP)		548	MOVQ AX, ret+16(FP)
	542	RET		549	RET
	543			550
	544	TEXT runtime·xchgp(SB), NOSPLIT, $0-24		551	TEXT runtime·xchgp(SB), NOSPLIT, $0-24
n	545	MOVQ ptr+0(FP), BX	n	552	MOVQ ptr+0(FP), BX
	546	MOVQ new+8(FP), AX		553	MOVQ new+8(FP), AX
	547	XCHGQ AX, 0(BX)		554	XCHGQ AX, 0(BX)
	548	MOVQ AX, ret+16(FP)		555	MOVQ AX, ret+16(FP)
	549	RET		556	RET
	550			557
	551	TEXT runtime·xchguintptr(SB), NOSPLIT, $0-24		558	TEXT runtime·xchguintptr(SB), NOSPLIT, $0-24
n	552	JMP runtime·xchg64(SB)	n	559	JMP runtime·xchg64(SB)
	553			560
n	554	TEXT runtime·procyield(SB),NOSPLIT,$0-0	n	561	TEXT runtime·procyield(SB), NOSPLIT, $0-0
	555	MOVL cycles+0(FP), AX		562	MOVL cycles+0(FP), AX
				563
	556	again:		564	again:
	557	PAUSE		565	PAUSE
n	558	SUBL $1, AX	n	566	SUBL $1, AX
	559	JNZ again		567	JNZ again
	560	RET		568	RET
	561			569
	562	TEXT runtime·atomicstorep(SB), NOSPLIT, $0-16		570	TEXT runtime·atomicstorep(SB), NOSPLIT, $0-16
n	563	MOVQ ptr+0(FP), BX	n	571	MOVQ ptr+0(FP), BX
	564	MOVQ val+8(FP), AX		572	MOVQ val+8(FP), AX
	565	XCHGQ AX, 0(BX)		573	XCHGQ AX, 0(BX)
	566	RET		574	RET
	567			575
	568	TEXT runtime·atomicstore(SB), NOSPLIT, $0-12		576	TEXT runtime·atomicstore(SB), NOSPLIT, $0-12
n	569	MOVQ ptr+0(FP), BX	n	577	MOVQ ptr+0(FP), BX
	570	MOVL val+8(FP), AX		578	MOVL val+8(FP), AX
	571	XCHGL AX, 0(BX)		579	XCHGL AX, 0(BX)
	572	RET		580	RET
	573			581
	574	TEXT runtime·atomicstore64(SB), NOSPLIT, $0-16		582	TEXT runtime·atomicstore64(SB), NOSPLIT, $0-16
n	575	MOVQ ptr+0(FP), BX	n	583	MOVQ ptr+0(FP), BX
	576	MOVQ val+8(FP), AX		584	MOVQ val+8(FP), AX
	577	XCHGQ AX, 0(BX)		585	XCHGQ AX, 0(BX)
	578	RET		586	RET
	579			587
	580	// void runtime·atomicor8(byte volatile*, byte);		588	// void runtime·atomicor8(byte volatile*, byte);
	581	TEXT runtime·atomicor8(SB), NOSPLIT, $0-9		589	TEXT runtime·atomicor8(SB), NOSPLIT, $0-9
n	582	MOVQ ptr+0(FP), AX	n	590	MOVQ ptr+0(FP), AX
	583	MOVB val+8(FP), BX		591	MOVB val+8(FP), BX
	584	LOCK		592	LOCK
n	585	ORB BX, (AX)	n	593	ORB BX, (AX)
	586	RET		594	RET
	587			595
	588	// void jmpdefer(fn, sp);		596	// void jmpdefer(fn, sp);
	589	// called from deferreturn.		597	// called from deferreturn.
	590	// 1. pop the caller		598	// 1. pop the caller
	591	// 2. sub 5 bytes from the callers return		599	// 2. sub 5 bytes from the callers return
	592	// 3. jmp to the argument		600	// 3. jmp to the argument
	593	TEXT runtime·jmpdefer(SB), NOSPLIT, $0-16		601	TEXT runtime·jmpdefer(SB), NOSPLIT, $0-16
n	594	MOVQ fv+0(FP), DX // fn	n	602	MOVQ fv+0(FP), DX // fn
	595	MOVQ argp+8(FP), BX // caller sp		603	MOVQ argp+8(FP), BX // caller sp
	596	LEAQ -8(BX), SP // caller sp after CALL		604	LEAQ -8(BX), SP // caller sp after CALL
	597	SUBQ $5, (SP) // return to CALL again		605	SUBQ $5, (SP) // return to CALL again
	598	MOVQ 0(DX), BX		606	MOVQ 0(DX), BX
	599	JMP BX // but first run the deferred function		607	JMP BX // but first run the deferred function
	600			608
	601	// Save state of caller into g->sched. Smashes R8, R9.		609	// Save state of caller into g->sched. Smashes R8, R9.
n	602	TEXT gosave<>(SB),NOSPLIT,$0	n	610	TEXT gosave<>(SB), NOSPLIT, $0
	603	get_tls(R8)		611	get_tls(R8)
n	604	MOVQ g(R8), R8	n	612	MOVQ g(R8), R8
	605	MOVQ 0(SP), R9		613	MOVQ 0(SP), R9
	606	MOVQ R9, (g_sched+gobuf_pc)(R8)		614	MOVQ R9, (g_sched+gobuf_pc)(R8)
	607	LEAQ 8(SP), R9		615	LEAQ 8(SP), R9
	608	MOVQ R9, (g_sched+gobuf_sp)(R8)		616	MOVQ R9, (g_sched+gobuf_sp)(R8)
	609	MOVQ $0, (g_sched+gobuf_ret)(R8)		617	MOVQ $0, (g_sched+gobuf_ret)(R8)
	610	MOVQ $0, (g_sched+gobuf_ctxt)(R8)		618	MOVQ $0, (g_sched+gobuf_ctxt)(R8)
	611	RET		619	RET
	612			620
	613	// asmcgocall(void(fn)(void), void *arg)		621	// asmcgocall(void(fn)(void), void *arg)
	614	// Call fn(arg) on the scheduler stack,		622	// Call fn(arg) on the scheduler stack,
	615	// aligned appropriately for the gcc ABI.		623	// aligned appropriately for the gcc ABI.
	616	// See cgocall.c for more details.		624	// See cgocall.c for more details.
n	617	TEXT ·asmcgocall(SB),NOSPLIT,$0-16	n	625	TEXT ·asmcgocall(SB), NOSPLIT, $0-16
	618	MOVQ fn+0(FP), AX		626	MOVQ fn+0(FP), AX
	619	MOVQ arg+8(FP), BX		627	MOVQ arg+8(FP), BX
	620	CALL asmcgocall<>(SB)		628	CALL asmcgocall<>(SB)
	621	RET		629	RET
	622			630
n	623	TEXT ·asmcgocall_errno(SB),NOSPLIT,$0-20	n	631	TEXT ·asmcgocall_errno(SB), NOSPLIT, $0-20
	624	MOVQ fn+0(FP), AX		632	MOVQ fn+0(FP), AX
	625	MOVQ arg+8(FP), BX		633	MOVQ arg+8(FP), BX
	626	CALL asmcgocall<>(SB)		634	CALL asmcgocall<>(SB)
	627	MOVL AX, ret+16(FP)		635	MOVL AX, ret+16(FP)
	628	RET		636	RET
	629			637
	630	// asmcgocall common code. fn in AX, arg in BX. returns errno in AX.		638	// asmcgocall common code. fn in AX, arg in BX. returns errno in AX.
n	631	TEXT asmcgocall<>(SB),NOSPLIT,$0-0	n	639	TEXT asmcgocall<>(SB), NOSPLIT, $0-0
	632	MOVQ SP, DX		640	MOVQ SP, DX
	633			641
	634	// Figure out if we need to switch to m->g0 stack.		642	// Figure out if we need to switch to m->g0 stack.
	635	// We get called to create new OS threads too, and those		643	// We get called to create new OS threads too, and those
	636	// come in on the m->g0 stack already.		644	// come in on the m->g0 stack already.
	637	get_tls(CX)		645	get_tls(CX)
n	638	MOVQ g(CX), BP	n	646	MOVQ g(CX), BP
	639	MOVQ g_m(BP), BP		647	MOVQ g_m(BP), BP
	640	MOVQ m_g0(BP), SI		648	MOVQ m_g0(BP), SI
	641	MOVQ g(CX), DI		649	MOVQ g(CX), DI
	642	CMPQ SI, DI		650	CMPQ SI, DI
	643	JEQ nosave		651	JEQ nosave
	644	MOVQ m_gsignal(BP), SI		652	MOVQ m_gsignal(BP), SI
	645	CMPQ SI, DI		653	CMPQ SI, DI
	646	JEQ nosave		654	JEQ nosave
	647			655
	648	MOVQ m_g0(BP), SI		656	MOVQ m_g0(BP), SI
	649	CALL gosave<>(SB)		657	CALL gosave<>(SB)
	650	MOVQ SI, g(CX)		658	MOVQ SI, g(CX)
	651	MOVQ (g_sched+gobuf_sp)(SI), SP		659	MOVQ (g_sched+gobuf_sp)(SI), SP
				660
	652	nosave:		661	nosave:
	653			662
	654	// Now on a scheduling stack (a pthread-created stack).		663	// Now on a scheduling stack (a pthread-created stack).
	655	// Make sure we have enough room for 4 stack-backed fast-call		664	// Make sure we have enough room for 4 stack-backed fast-call
	656	// registers as per windows amd64 calling convention.		665	// registers as per windows amd64 calling convention.
n	657	SUBQ $64, SP	n	666	SUBQ $64, SP
	658	ANDQ $~15, SP // alignment for gcc ABI		667	ANDQ $~15, SP // alignment for gcc ABI
	659	MOVQ DI, 48(SP) // save g		668	MOVQ DI, 48(SP) // save g
	660	MOVQ (g_stack+stack_hi)(DI), DI		669	MOVQ (g_stack+stack_hi)(DI), DI
	661	SUBQ DX, DI		670	SUBQ DX, DI
	662	MOVQ DI, 40(SP) // save depth in stack (can't just save SP, as stack might be copied during a callback)		671	MOVQ DI, 40(SP) // save depth in stack (can't just save SP, as stack might be copied during a callback)
	663	MOVQ BX, DI // DI = first argument in AMD64 ABI		672	MOVQ BX, DI // DI = first argument in AMD64 ABI
	664	MOVQ BX, CX // CX = first argument in Win64		673	MOVQ BX, CX // CX = first argument in Win64
	665	CALL AX		674	CALL AX
	666			675
	667	// Restore registers, g, stack pointer.		676	// Restore registers, g, stack pointer.
	668	get_tls(CX)		677	get_tls(CX)
n	669	MOVQ 48(SP), DI	n	678	MOVQ 48(SP), DI
	670	MOVQ (g_stack+stack_hi)(DI), SI		679	MOVQ (g_stack+stack_hi)(DI), SI
	671	SUBQ 40(SP), SI		680	SUBQ 40(SP), SI
	672	MOVQ DI, g(CX)		681	MOVQ DI, g(CX)
	673	MOVQ SI, SP		682	MOVQ SI, SP
	674	RET		683	RET
	675			684
	676	// cgocallback(void (fn)(void), void *frame, uintptr framesize)		685	// cgocallback(void (fn)(void), void *frame, uintptr framesize)
	677	// Turn the fn into a Go func (by taking its address) and call		686	// Turn the fn into a Go func (by taking its address) and call
	678	// cgocallback_gofunc.		687	// cgocallback_gofunc.
n	679	TEXT runtime·cgocallback(SB),NOSPLIT,$24-24	n	688	TEXT runtime·cgocallback(SB), NOSPLIT, $24-24
	680	LEAQ fn+0(FP), AX		689	LEAQ fn+0(FP), AX
	681	MOVQ AX, 0(SP)		690	MOVQ AX, 0(SP)
	682	MOVQ frame+8(FP), AX		691	MOVQ frame+8(FP), AX
	683	MOVQ AX, 8(SP)		692	MOVQ AX, 8(SP)
	684	MOVQ framesize+16(FP), AX		693	MOVQ framesize+16(FP), AX
	685	MOVQ AX, 16(SP)		694	MOVQ AX, 16(SP)
	686	MOVQ $runtime·cgocallback_gofunc(SB), AX		695	MOVQ $runtime·cgocallback_gofunc(SB), AX
	687	CALL AX		696	CALL AX
	688	RET		697	RET
	689			698
	690	// cgocallback_gofunc(FuncVal, void frame, uintptr framesize)		699	// cgocallback_gofunc(FuncVal, void frame, uintptr framesize)
	691	// See cgocall.c for more details.		700	// See cgocall.c for more details.
n	692	TEXT ·cgocallback_gofunc(SB),NOSPLIT,$8-24	n	701	TEXT ·cgocallback_gofunc(SB), NOSPLIT, $8-24
	693	NO_LOCAL_POINTERS		702	NO_LOCAL_POINTERS
	694			703
	695	// If g is nil, Go did not create the current thread.		704	// If g is nil, Go did not create the current thread.
	696	// Call needm to obtain one m for temporary use.		705	// Call needm to obtain one m for temporary use.
	697	// In this case, we're running on the thread stack, so there's		706	// In this case, we're running on the thread stack, so there's
	698	// lots of space, but the linker doesn't know. Hide the call from		707	// lots of space, but the linker doesn't know. Hide the call from
	699	// the linker analysis by using an indirect call through AX.		708	// the linker analysis by using an indirect call through AX.
	700	get_tls(CX)		709	get_tls(CX)
n			n	710
	701	#ifdef GOOS_windows		711	#ifdef GOOS_windows
n	702	MOVL $0, BP	n	712	MOVL $0, BP
	703	CMPQ CX, $0		713	CMPQ CX, $0
	704	JEQ 2(PC)		714	JEQ 2(PC)
				715
	705	#endif		716	#endif
n	706	MOVQ g(CX), BP	n	717	MOVQ g(CX), BP
	707	CMPQ BP, $0		718	CMPQ BP, $0
	708	JEQ needm		719	JEQ needm
	709	MOVQ g_m(BP), BP		720	MOVQ g_m(BP), BP
	710	MOVQ BP, R8 // holds oldm until end of function		721	MOVQ BP, R8 // holds oldm until end of function
	711	JMP havem		722	JMP havem
				723
	712	needm:		724	needm:
n	713	MOVQ $0, 0(SP)	n	725	MOVQ $0, 0(SP)
	714	MOVQ $runtime·needm(SB), AX		726	MOVQ $runtime·needm(SB), AX
	715	CALL AX		727	CALL AX
	716	MOVQ 0(SP), R8		728	MOVQ 0(SP), R8
	717	get_tls(CX)		729	get_tls(CX)
n	718	MOVQ g(CX), BP	n	730	MOVQ g(CX), BP
	719	MOVQ g_m(BP), BP		731	MOVQ g_m(BP), BP
	720			732
	721	// Set m->sched.sp = SP, so that if a panic happens		733	// Set m->sched.sp = SP, so that if a panic happens
	722	// during the function we are about to execute, it will		734	// during the function we are about to execute, it will
	723	// have a valid SP to run on the g0 stack.		735	// have a valid SP to run on the g0 stack.
	724	// The next few lines (after the havem label)		736	// The next few lines (after the havem label)
	725	// will save this SP onto the stack and then write		737	// will save this SP onto the stack and then write
	726	// the same SP back to m->sched.sp. That seems redundant,		738	// the same SP back to m->sched.sp. That seems redundant,
	727	// but if an unrecovered panic happens, unwindm will		739	// but if an unrecovered panic happens, unwindm will
	728	// restore the g->sched.sp from the stack location		740	// restore the g->sched.sp from the stack location
	729	// and then onM will try to use it. If we don't set it here,		741	// and then onM will try to use it. If we don't set it here,
	730	// that restored SP will be uninitialized (typically 0) and		742	// that restored SP will be uninitialized (typically 0) and
	731	// will not be usable.		743	// will not be usable.
n	732	MOVQ m_g0(BP), SI	n	744	MOVQ m_g0(BP), SI
	733	MOVQ SP, (g_sched+gobuf_sp)(SI)		745	MOVQ SP, (g_sched+gobuf_sp)(SI)
	734			746
	735	havem:		747	havem:
	736	// Now there's a valid m, and we're running on its m->g0.		748	// Now there's a valid m, and we're running on its m->g0.
	737	// Save current m->g0->sched.sp on stack and then set it to SP.		749	// Save current m->g0->sched.sp on stack and then set it to SP.
	738	// Save current sp in m->g0->sched.sp in preparation for		750	// Save current sp in m->g0->sched.sp in preparation for
	739	// switch back to m->curg stack.		751	// switch back to m->curg stack.
	740	// NOTE: unwindm knows that the saved g->sched.sp is at 0(SP).		752	// NOTE: unwindm knows that the saved g->sched.sp is at 0(SP).
n	741	MOVQ m_g0(BP), SI	n	753	MOVQ m_g0(BP), SI
	742	MOVQ (g_sched+gobuf_sp)(SI), AX		754	MOVQ (g_sched+gobuf_sp)(SI), AX
	743	MOVQ AX, 0(SP)		755	MOVQ AX, 0(SP)
	744	MOVQ SP, (g_sched+gobuf_sp)(SI)		756	MOVQ SP, (g_sched+gobuf_sp)(SI)
	745			757
	746	// Switch to m->curg stack and call runtime.cgocallbackg.		758	// Switch to m->curg stack and call runtime.cgocallbackg.
	747	// Because we are taking over the execution of m->curg		759	// Because we are taking over the execution of m->curg
	748	// but not resuming what had been running, we need to		760	// but not resuming what had been running, we need to
	749	// save that information (m->curg->sched) so we can restore it.		761	// save that information (m->curg->sched) so we can restore it.
	750	// We can restore m->curg->sched.sp easily, because calling		762	// We can restore m->curg->sched.sp easily, because calling
	751	// runtime.cgocallbackg leaves SP unchanged upon return.		763	// runtime.cgocallbackg leaves SP unchanged upon return.
	752	// To save m->curg->sched.pc, we push it onto the stack.		764	// To save m->curg->sched.pc, we push it onto the stack.
	753	// This has the added benefit that it looks to the traceback		765	// This has the added benefit that it looks to the traceback
	754	// routine like cgocallbackg is going to return to that		766	// routine like cgocallbackg is going to return to that
	755	// PC (because the frame we allocate below has the same		767	// PC (because the frame we allocate below has the same
	756	// size as cgocallback_gofunc's frame declared above)		768	// size as cgocallback_gofunc's frame declared above)
	757	// so that the traceback will seamlessly trace back into		769	// so that the traceback will seamlessly trace back into
	758	// the earlier calls.		770	// the earlier calls.
	759	//		771	//
	760	// In the new goroutine, 0(SP) holds the saved R8.		772	// In the new goroutine, 0(SP) holds the saved R8.
n	761	MOVQ m_curg(BP), SI	n	773	MOVQ m_curg(BP), SI
	762	MOVQ SI, g(CX)		774	MOVQ SI, g(CX)
	763	MOVQ (g_sched+gobuf_sp)(SI), DI // prepare stack as DI		775	MOVQ (g_sched+gobuf_sp)(SI), DI // prepare stack as DI
	764	MOVQ (g_sched+gobuf_pc)(SI), BP		776	MOVQ (g_sched+gobuf_pc)(SI), BP
	765	MOVQ BP, -8(DI)		777	MOVQ BP, -8(DI)
	766	LEAQ -(8+8)(DI), SP		778	LEAQ -(8+8)(DI), SP
	767	MOVQ R8, 0(SP)		779	MOVQ R8, 0(SP)
	768	CALL runtime·cgocallbackg(SB)		780	CALL runtime·cgocallbackg(SB)
	769	MOVQ 0(SP), R8		781	MOVQ 0(SP), R8
	770			782
	771	// Restore g->sched (== m->curg->sched) from saved values.		783	// Restore g->sched (== m->curg->sched) from saved values.
	772	get_tls(CX)		784	get_tls(CX)
n	773	MOVQ g(CX), SI	n	785	MOVQ g(CX), SI
	774	MOVQ 8(SP), BP		786	MOVQ 8(SP), BP
	775	MOVQ BP, (g_sched+gobuf_pc)(SI)		787	MOVQ BP, (g_sched+gobuf_pc)(SI)
	776	LEAQ (8+8)(SP), DI		788	LEAQ (8+8)(SP), DI
	777	MOVQ DI, (g_sched+gobuf_sp)(SI)		789	MOVQ DI, (g_sched+gobuf_sp)(SI)
	778			790
	779	// Switch back to m->g0's stack and restore m->g0->sched.sp.		791	// Switch back to m->g0's stack and restore m->g0->sched.sp.
	780	// (Unlike m->curg, the g0 goroutine never uses sched.pc,		792	// (Unlike m->curg, the g0 goroutine never uses sched.pc,
	781	// so we do not have to restore it.)		793	// so we do not have to restore it.)
n	782	MOVQ g(CX), BP	n	794	MOVQ g(CX), BP
	783	MOVQ g_m(BP), BP		795	MOVQ g_m(BP), BP
	784	MOVQ m_g0(BP), SI		796	MOVQ m_g0(BP), SI
	785	MOVQ SI, g(CX)		797	MOVQ SI, g(CX)
	786	MOVQ (g_sched+gobuf_sp)(SI), SP		798	MOVQ (g_sched+gobuf_sp)(SI), SP
	787	MOVQ 0(SP), AX		799	MOVQ 0(SP), AX
	788	MOVQ AX, (g_sched+gobuf_sp)(SI)		800	MOVQ AX, (g_sched+gobuf_sp)(SI)
	789			801
	790	// If the m on entry was nil, we called needm above to borrow an m		802	// If the m on entry was nil, we called needm above to borrow an m
	791	// for the duration of the call. Since the call is over, return it with dropm.		803	// for the duration of the call. Since the call is over, return it with dropm.
n	792	CMPQ R8, $0	n	804	CMPQ R8, $0
	793	JNE 3(PC)		805	JNE 3(PC)
	794	MOVQ $runtime·dropm(SB), AX		806	MOVQ $runtime·dropm(SB), AX
	795	CALL AX		807	CALL AX
	796			808
	797	// Done!		809	// Done!
	798	RET		810	RET
	799			811
	800	// void setg(G*); set g. for use by needm.		812	// void setg(G*); set g. for use by needm.
	801	TEXT runtime·setg(SB), NOSPLIT, $0-8		813	TEXT runtime·setg(SB), NOSPLIT, $0-8
n	802	MOVQ gg+0(FP), BX	n	814	MOVQ gg+0(FP), BX
				815
	803	#ifdef GOOS_windows		816	#ifdef GOOS_windows
n	804	CMPQ BX, $0	n	817	CMPQ BX, $0
	805	JNE settls		818	JNE settls
	806	MOVQ $0, 0x28(GS)		819	MOVQ $0, 0x28(GS)
	807	RET		820	RET
n			n	821
	808	settls:		822	settls:
n	809	MOVQ g_m(BX), AX	n	823	MOVQ g_m(BX), AX
	810	LEAQ m_tls(AX), AX		824	LEAQ m_tls(AX), AX
	811	MOVQ AX, 0x28(GS)		825	MOVQ AX, 0x28(GS)
				826
	812	#endif		827	#endif
	813	get_tls(CX)		828	get_tls(CX)
n	814	MOVQ BX, g(CX)	n	829	MOVQ BX, g(CX)
	815	RET		830	RET
	816			831
	817	// void setg_gcc(G*); set g called from gcc.		832	// void setg_gcc(G*); set g called from gcc.
n	818	TEXT setg_gcc<>(SB),NOSPLIT,$0	n	833	TEXT setg_gcc<>(SB), NOSPLIT, $0
	819	get_tls(AX)		834	get_tls(AX)
n	820	MOVQ DI, g(AX)	n	835	MOVQ DI, g(AX)
	821	RET		836	RET
	822			837
	823	// check that SP is in range [g->stack.lo, g->stack.hi)		838	// check that SP is in range [g->stack.lo, g->stack.hi)
	824	TEXT runtime·stackcheck(SB), NOSPLIT, $0-0		839	TEXT runtime·stackcheck(SB), NOSPLIT, $0-0
	825	get_tls(CX)		840	get_tls(CX)
n	826	MOVQ g(CX), AX	n	841	MOVQ g(CX), AX
	827	CMPQ (g_stack+stack_hi)(AX), SP		842	CMPQ (g_stack+stack_hi)(AX), SP
	828	JHI 2(PC)		843	JHI 2(PC)
	829	INT $3		844	INT $3
	830	CMPQ SP, (g_stack+stack_lo)(AX)		845	CMPQ SP, (g_stack+stack_lo)(AX)
	831	JHI 2(PC)		846	JHI 2(PC)
	832	INT $3		847	INT $3
	833	RET		848	RET
	834			849
n	835	TEXT runtime·getcallerpc(SB),NOSPLIT,$0-16	n	850	TEXT runtime·getcallerpc(SB), NOSPLIT, $0-16
	836	MOVQ argp+0(FP),AX // addr of first arg		851	MOVQ argp+0(FP), AX // addr of first arg
	837	MOVQ -8(AX),AX // get calling pc		852	MOVQ -8(AX), AX // get calling pc
	838	MOVQ AX, ret+8(FP)		853	MOVQ AX, ret+8(FP)
	839	RET		854	RET
	840			855
n	841	TEXT runtime·gogetcallerpc(SB),NOSPLIT,$0-16	n	856	TEXT runtime·gogetcallerpc(SB), NOSPLIT, $0-16
	842	MOVQ p+0(FP),AX // addr of first arg		857	MOVQ p+0(FP), AX // addr of first arg
	843	MOVQ -8(AX),AX // get calling pc		858	MOVQ -8(AX), AX // get calling pc
	844	MOVQ AX,ret+8(FP)		859	MOVQ AX, ret+8(FP)
	845	RET		860	RET
	846			861
n	847	TEXT runtime·setcallerpc(SB),NOSPLIT,$0-16	n	862	TEXT runtime·setcallerpc(SB), NOSPLIT, $0-16
	848	MOVQ argp+0(FP),AX // addr of first arg		863	MOVQ argp+0(FP), AX // addr of first arg
	849	MOVQ pc+8(FP), BX		864	MOVQ pc+8(FP), BX
	850	MOVQ BX, -8(AX) // set calling pc		865	MOVQ BX, -8(AX) // set calling pc
	851	RET		866	RET
	852			867
n	853	TEXT runtime·getcallersp(SB),NOSPLIT,$0-16	n	868	TEXT runtime·getcallersp(SB), NOSPLIT, $0-16
	854	MOVQ argp+0(FP), AX		869	MOVQ argp+0(FP), AX
	855	MOVQ AX, ret+8(FP)		870	MOVQ AX, ret+8(FP)
	856	RET		871	RET
	857			872
	858	// func gogetcallersp(p unsafe.Pointer) uintptr		873	// func gogetcallersp(p unsafe.Pointer) uintptr
n	859	TEXT runtime·gogetcallersp(SB),NOSPLIT,$0-16	n	874	TEXT runtime·gogetcallersp(SB), NOSPLIT, $0-16
	860	MOVQ p+0(FP),AX // addr of first arg		875	MOVQ p+0(FP), AX // addr of first arg
	861	MOVQ AX, ret+8(FP)		876	MOVQ AX, ret+8(FP)
	862	RET		877	RET
	863			878
	864	// int64 runtime·cputicks(void)		879	// int64 runtime·cputicks(void)
n	865	TEXT runtime·cputicks(SB),NOSPLIT,$0-0	n	880	TEXT runtime·cputicks(SB), NOSPLIT, $0-0
	866	RDTSC		881	RDTSC
n	867	SHLQ $32, DX	n	882	SHLQ $32, DX
	868	ADDQ DX, AX		883	ADDQ DX, AX
	869	MOVQ AX, ret+0(FP)		884	MOVQ AX, ret+0(FP)
	870	RET		885	RET
	871			886
	872	// hash function using AES hardware instructions		887	// hash function using AES hardware instructions
n	873	TEXT runtime·aeshash(SB),NOSPLIT,$0-32	n	888	TEXT runtime·aeshash(SB), NOSPLIT, $0-32
	874	MOVQ p+0(FP), AX // ptr to data		889	MOVQ p+0(FP), AX // ptr to data
	875	MOVQ s+8(FP), CX // size		890	MOVQ s+8(FP), CX // size
	876	JMP runtime·aeshashbody(SB)		891	JMP runtime·aeshashbody(SB)
	877			892
n	878	TEXT runtime·aeshashstr(SB),NOSPLIT,$0-32	n	893	TEXT runtime·aeshashstr(SB), NOSPLIT, $0-32
	879	MOVQ p+0(FP), AX // ptr to string struct		894	MOVQ p+0(FP), AX // ptr to string struct
				895
	880	// s+8(FP) is ignored, it is always sizeof(String)		896	// s+8(FP) is ignored, it is always sizeof(String)
n	881	MOVQ 8(AX), CX // length of string	n	897	MOVQ 8(AX), CX // length of string
	882	MOVQ (AX), AX // string data		898	MOVQ (AX), AX // string data
	883	JMP runtime·aeshashbody(SB)		899	JMP runtime·aeshashbody(SB)
	884			900
	885	// AX: data		901	// AX: data
	886	// CX: length		902	// CX: length
n	887	TEXT runtime·aeshashbody(SB),NOSPLIT,$0-32	n	903	TEXT runtime·aeshashbody(SB), NOSPLIT, $0-32
	888	MOVQ h+16(FP), X0 // seed to low 64 bits of xmm0		904	MOVQ h+16(FP), X0 // seed to low 64 bits of xmm0
	889	PINSRQ $1, CX, X0 // size to high 64 bits of xmm0		905	PINSRQ $1, CX, X0 // size to high 64 bits of xmm0
	890	MOVO runtime·aeskeysched+0(SB), X2		906	MOVO runtime·aeskeysched+0(SB), X2
	891	MOVO runtime·aeskeysched+16(SB), X3		907	MOVO runtime·aeskeysched+16(SB), X3
	892	CMPQ CX, $16		908	CMPQ CX, $16
	893	JB aessmall		909	JB aessmall
				910
	894	aesloop:		911	aesloop:
n	895	CMPQ CX, $16	n	912	CMPQ CX, $16
	896	JBE aesloopend		913	JBE aesloopend
	897	MOVOU (AX), X1		914	MOVOU (AX), X1
	898	AESENC X2, X0		915	AESENC X2, X0
	899	AESENC X1, X0		916	AESENC X1, X0
	900	SUBQ $16, CX		917	SUBQ $16, CX
	901	ADDQ $16, AX		918	ADDQ $16, AX
	902	JMP aesloop		919	JMP aesloop
				920
	903	// 1-16 bytes remaining		921	// 1-16 bytes remaining
	904	aesloopend:		922	aesloopend:
	905	// This load may overlap with the previous load above.		923	// This load may overlap with the previous load above.
	906	// We'll hash some bytes twice, but that's ok.		924	// We'll hash some bytes twice, but that's ok.
n	907	MOVOU -16(AX)(CX*1), X1	n	925	MOVOU -16(AX)(CX*1), X1
	908	JMP partial		926	JMP partial
				927
	909	// 0-15 bytes		928	// 0-15 bytes
	910	aessmall:		929	aessmall:
n	911	TESTQ CX, CX	n	930	TESTQ CX, CX
	912	JE finalize // 0 bytes		931	JE finalize // 0 bytes
	913			932
n	914	CMPB AX, $0xf0	n	933	CMPB AX, $0xf0
	915	JA highpartial		934	JA highpartial
	916			935
	917	// 16 bytes loaded at this address won't cross		936	// 16 bytes loaded at this address won't cross
	918	// a page boundary, so we can load it directly.		937	// a page boundary, so we can load it directly.
n	919	MOVOU (AX), X1	n	938	MOVOU (AX), X1
	920	ADDQ CX, CX		939	ADDQ CX, CX
	921	MOVQ $masks<>(SB), BP		940	MOVQ $masks<>(SB), BP
	922	PAND (BP)(CX*8), X1		941	PAND (BP)(CX*8), X1
	923	JMP partial		942	JMP partial
				943
	924	highpartial:		944	highpartial:
	925	// address ends in 1111xxxx. Might be up against		945	// address ends in 1111xxxx. Might be up against
	926	// a page boundary, so load ending at last byte.		946	// a page boundary, so load ending at last byte.
	927	// Then shift bytes down using pshufb.		947	// Then shift bytes down using pshufb.
n	928	MOVOU -16(AX)(CX*1), X1	n	948	MOVOU -16(AX)(CX*1), X1
	929	ADDQ CX, CX		949	ADDQ CX, CX
	930	MOVQ $shifts<>(SB), BP		950	MOVQ $shifts<>(SB), BP
	931	PSHUFB (BP)(CX*8), X1		951	PSHUFB (BP)(CX*8), X1
				952
	932	partial:		953	partial:
	933	// incorporate partial block into hash		954	// incorporate partial block into hash
n	934	AESENC X3, X0	n	955	AESENC X3, X0
	935	AESENC X1, X0		956	AESENC X1, X0
				957
	936	finalize:		958	finalize:
	937	// finalize hash		959	// finalize hash
n	938	AESENC X2, X0	n	960	AESENC X2, X0
	939	AESENC X3, X0		961	AESENC X3, X0
	940	AESENC X2, X0		962	AESENC X2, X0
	941	MOVQ X0, res+24(FP)		963	MOVQ X0, res+24(FP)
	942	RET		964	RET
	943			965
n	944	TEXT runtime·aeshash32(SB),NOSPLIT,$0-32	n	966	TEXT runtime·aeshash32(SB), NOSPLIT, $0-32
	945	MOVQ p+0(FP), AX // ptr to data		967	MOVQ p+0(FP), AX // ptr to data
				968
	946	// s+8(FP) is ignored, it is always sizeof(int32)		969	// s+8(FP) is ignored, it is always sizeof(int32)
n	947	MOVQ h+16(FP), X0 // seed	n	970	MOVQ h+16(FP), X0 // seed
	948	PINSRD $2, (AX), X0 // data		971	PINSRD $2, (AX), X0 // data
	949	AESENC runtime·aeskeysched+0(SB), X0		972	AESENC runtime·aeskeysched+0(SB), X0
	950	AESENC runtime·aeskeysched+16(SB), X0		973	AESENC runtime·aeskeysched+16(SB), X0
	951	AESENC runtime·aeskeysched+0(SB), X0		974	AESENC runtime·aeskeysched+0(SB), X0
	952	MOVQ X0, ret+24(FP)		975	MOVQ X0, ret+24(FP)
	953	RET		976	RET
	954			977
n	955	TEXT runtime·aeshash64(SB),NOSPLIT,$0-32	n	978	TEXT runtime·aeshash64(SB), NOSPLIT, $0-32
	956	MOVQ p+0(FP), AX // ptr to data		979	MOVQ p+0(FP), AX // ptr to data
				980
	957	// s+8(FP) is ignored, it is always sizeof(int64)		981	// s+8(FP) is ignored, it is always sizeof(int64)
n	958	MOVQ h+16(FP), X0 // seed	n	982	MOVQ h+16(FP), X0 // seed
	959	PINSRQ $1, (AX), X0 // data		983	PINSRQ $1, (AX), X0 // data
	960	AESENC runtime·aeskeysched+0(SB), X0		984	AESENC runtime·aeskeysched+0(SB), X0
	961	AESENC runtime·aeskeysched+16(SB), X0		985	AESENC runtime·aeskeysched+16(SB), X0
	962	AESENC runtime·aeskeysched+0(SB), X0		986	AESENC runtime·aeskeysched+0(SB), X0
	963	MOVQ X0, ret+24(FP)		987	MOVQ X0, ret+24(FP)
	964	RET		988	RET
	965			989
	966	// simple mask to get rid of data in the high part of the register.		990	// simple mask to get rid of data in the high part of the register.
	967	DATA masks<>+0x00(SB)/8, $0x0000000000000000		991	DATA masks<>+0x00(SB)/8, $0x0000000000000000
	968	DATA masks<>+0x08(SB)/8, $0x0000000000000000		992	DATA masks<>+0x08(SB)/8, $0x0000000000000000
	969	DATA masks<>+0x10(SB)/8, $0x00000000000000ff		993	DATA masks<>+0x10(SB)/8, $0x00000000000000ff
	970	DATA masks<>+0x18(SB)/8, $0x0000000000000000		994	DATA masks<>+0x18(SB)/8, $0x0000000000000000
	971	DATA masks<>+0x20(SB)/8, $0x000000000000ffff		995	DATA masks<>+0x20(SB)/8, $0x000000000000ffff
	972	DATA masks<>+0x28(SB)/8, $0x0000000000000000		996	DATA masks<>+0x28(SB)/8, $0x0000000000000000
	973	DATA masks<>+0x30(SB)/8, $0x0000000000ffffff		997	DATA masks<>+0x30(SB)/8, $0x0000000000ffffff
	974	DATA masks<>+0x38(SB)/8, $0x0000000000000000		998	DATA masks<>+0x38(SB)/8, $0x0000000000000000
	975	DATA masks<>+0x40(SB)/8, $0x00000000ffffffff		999	DATA masks<>+0x40(SB)/8, $0x00000000ffffffff
	976	DATA masks<>+0x48(SB)/8, $0x0000000000000000		1000	DATA masks<>+0x48(SB)/8, $0x0000000000000000
	977	DATA masks<>+0x50(SB)/8, $0x000000ffffffffff		1001	DATA masks<>+0x50(SB)/8, $0x000000ffffffffff
	978	DATA masks<>+0x58(SB)/8, $0x0000000000000000		1002	DATA masks<>+0x58(SB)/8, $0x0000000000000000
	979	DATA masks<>+0x60(SB)/8, $0x0000ffffffffffff		1003	DATA masks<>+0x60(SB)/8, $0x0000ffffffffffff
	980	DATA masks<>+0x68(SB)/8, $0x0000000000000000		1004	DATA masks<>+0x68(SB)/8, $0x0000000000000000
	981	DATA masks<>+0x70(SB)/8, $0x00ffffffffffffff		1005	DATA masks<>+0x70(SB)/8, $0x00ffffffffffffff
	982	DATA masks<>+0x78(SB)/8, $0x0000000000000000		1006	DATA masks<>+0x78(SB)/8, $0x0000000000000000
	983	DATA masks<>+0x80(SB)/8, $0xffffffffffffffff		1007	DATA masks<>+0x80(SB)/8, $0xffffffffffffffff
	984	DATA masks<>+0x88(SB)/8, $0x0000000000000000		1008	DATA masks<>+0x88(SB)/8, $0x0000000000000000
	985	DATA masks<>+0x90(SB)/8, $0xffffffffffffffff		1009	DATA masks<>+0x90(SB)/8, $0xffffffffffffffff
	986	DATA masks<>+0x98(SB)/8, $0x00000000000000ff		1010	DATA masks<>+0x98(SB)/8, $0x00000000000000ff
	987	DATA masks<>+0xa0(SB)/8, $0xffffffffffffffff		1011	DATA masks<>+0xa0(SB)/8, $0xffffffffffffffff
	988	DATA masks<>+0xa8(SB)/8, $0x000000000000ffff		1012	DATA masks<>+0xa8(SB)/8, $0x000000000000ffff
	989	DATA masks<>+0xb0(SB)/8, $0xffffffffffffffff		1013	DATA masks<>+0xb0(SB)/8, $0xffffffffffffffff
	990	DATA masks<>+0xb8(SB)/8, $0x0000000000ffffff		1014	DATA masks<>+0xb8(SB)/8, $0x0000000000ffffff
	991	DATA masks<>+0xc0(SB)/8, $0xffffffffffffffff		1015	DATA masks<>+0xc0(SB)/8, $0xffffffffffffffff
	992	DATA masks<>+0xc8(SB)/8, $0x00000000ffffffff		1016	DATA masks<>+0xc8(SB)/8, $0x00000000ffffffff
	993	DATA masks<>+0xd0(SB)/8, $0xffffffffffffffff		1017	DATA masks<>+0xd0(SB)/8, $0xffffffffffffffff
	994	DATA masks<>+0xd8(SB)/8, $0x000000ffffffffff		1018	DATA masks<>+0xd8(SB)/8, $0x000000ffffffffff
	995	DATA masks<>+0xe0(SB)/8, $0xffffffffffffffff		1019	DATA masks<>+0xe0(SB)/8, $0xffffffffffffffff
	996	DATA masks<>+0xe8(SB)/8, $0x0000ffffffffffff		1020	DATA masks<>+0xe8(SB)/8, $0x0000ffffffffffff
	997	DATA masks<>+0xf0(SB)/8, $0xffffffffffffffff		1021	DATA masks<>+0xf0(SB)/8, $0xffffffffffffffff
	998	DATA masks<>+0xf8(SB)/8, $0x00ffffffffffffff		1022	DATA masks<>+0xf8(SB)/8, $0x00ffffffffffffff
n	999	GLOBL masks<>(SB),RODATA,$256	n	1023	GLOBL masks<>(SB), RODATA, $256
	1000			1024
	1001	// these are arguments to pshufb. They move data down from		1025	// these are arguments to pshufb. They move data down from
	1002	// the high bytes of the register to the low bytes of the register.		1026	// the high bytes of the register to the low bytes of the register.
	1003	// index is how many bytes to move.		1027	// index is how many bytes to move.
	1004	DATA shifts<>+0x00(SB)/8, $0x0000000000000000		1028	DATA shifts<>+0x00(SB)/8, $0x0000000000000000
	1005	DATA shifts<>+0x08(SB)/8, $0x0000000000000000		1029	DATA shifts<>+0x08(SB)/8, $0x0000000000000000
	1006	DATA shifts<>+0x10(SB)/8, $0xffffffffffffff0f		1030	DATA shifts<>+0x10(SB)/8, $0xffffffffffffff0f
	1007	DATA shifts<>+0x18(SB)/8, $0xffffffffffffffff		1031	DATA shifts<>+0x18(SB)/8, $0xffffffffffffffff
	1008	DATA shifts<>+0x20(SB)/8, $0xffffffffffff0f0e		1032	DATA shifts<>+0x20(SB)/8, $0xffffffffffff0f0e
	1009	DATA shifts<>+0x28(SB)/8, $0xffffffffffffffff		1033	DATA shifts<>+0x28(SB)/8, $0xffffffffffffffff
	1010	DATA shifts<>+0x30(SB)/8, $0xffffffffff0f0e0d		1034	DATA shifts<>+0x30(SB)/8, $0xffffffffff0f0e0d
	1011	DATA shifts<>+0x38(SB)/8, $0xffffffffffffffff		1035	DATA shifts<>+0x38(SB)/8, $0xffffffffffffffff
	1012	DATA shifts<>+0x40(SB)/8, $0xffffffff0f0e0d0c		1036	DATA shifts<>+0x40(SB)/8, $0xffffffff0f0e0d0c
	1013	DATA shifts<>+0x48(SB)/8, $0xffffffffffffffff		1037	DATA shifts<>+0x48(SB)/8, $0xffffffffffffffff
	1014	DATA shifts<>+0x50(SB)/8, $0xffffff0f0e0d0c0b		1038	DATA shifts<>+0x50(SB)/8, $0xffffff0f0e0d0c0b
	1015	DATA shifts<>+0x58(SB)/8, $0xffffffffffffffff		1039	DATA shifts<>+0x58(SB)/8, $0xffffffffffffffff
	1016	DATA shifts<>+0x60(SB)/8, $0xffff0f0e0d0c0b0a		1040	DATA shifts<>+0x60(SB)/8, $0xffff0f0e0d0c0b0a
	1017	DATA shifts<>+0x68(SB)/8, $0xffffffffffffffff		1041	DATA shifts<>+0x68(SB)/8, $0xffffffffffffffff
	1018	DATA shifts<>+0x70(SB)/8, $0xff0f0e0d0c0b0a09		1042	DATA shifts<>+0x70(SB)/8, $0xff0f0e0d0c0b0a09
	1019	DATA shifts<>+0x78(SB)/8, $0xffffffffffffffff		1043	DATA shifts<>+0x78(SB)/8, $0xffffffffffffffff
	1020	DATA shifts<>+0x80(SB)/8, $0x0f0e0d0c0b0a0908		1044	DATA shifts<>+0x80(SB)/8, $0x0f0e0d0c0b0a0908
	1021	DATA shifts<>+0x88(SB)/8, $0xffffffffffffffff		1045	DATA shifts<>+0x88(SB)/8, $0xffffffffffffffff
	1022	DATA shifts<>+0x90(SB)/8, $0x0e0d0c0b0a090807		1046	DATA shifts<>+0x90(SB)/8, $0x0e0d0c0b0a090807
	1023	DATA shifts<>+0x98(SB)/8, $0xffffffffffffff0f		1047	DATA shifts<>+0x98(SB)/8, $0xffffffffffffff0f
	1024	DATA shifts<>+0xa0(SB)/8, $0x0d0c0b0a09080706		1048	DATA shifts<>+0xa0(SB)/8, $0x0d0c0b0a09080706
	1025	DATA shifts<>+0xa8(SB)/8, $0xffffffffffff0f0e		1049	DATA shifts<>+0xa8(SB)/8, $0xffffffffffff0f0e
	1026	DATA shifts<>+0xb0(SB)/8, $0x0c0b0a0908070605		1050	DATA shifts<>+0xb0(SB)/8, $0x0c0b0a0908070605
	1027	DATA shifts<>+0xb8(SB)/8, $0xffffffffff0f0e0d		1051	DATA shifts<>+0xb8(SB)/8, $0xffffffffff0f0e0d
	1028	DATA shifts<>+0xc0(SB)/8, $0x0b0a090807060504		1052	DATA shifts<>+0xc0(SB)/8, $0x0b0a090807060504
	1029	DATA shifts<>+0xc8(SB)/8, $0xffffffff0f0e0d0c		1053	DATA shifts<>+0xc8(SB)/8, $0xffffffff0f0e0d0c
	1030	DATA shifts<>+0xd0(SB)/8, $0x0a09080706050403		1054	DATA shifts<>+0xd0(SB)/8, $0x0a09080706050403
	1031	DATA shifts<>+0xd8(SB)/8, $0xffffff0f0e0d0c0b		1055	DATA shifts<>+0xd8(SB)/8, $0xffffff0f0e0d0c0b
	1032	DATA shifts<>+0xe0(SB)/8, $0x0908070605040302		1056	DATA shifts<>+0xe0(SB)/8, $0x0908070605040302
	1033	DATA shifts<>+0xe8(SB)/8, $0xffff0f0e0d0c0b0a		1057	DATA shifts<>+0xe8(SB)/8, $0xffff0f0e0d0c0b0a
	1034	DATA shifts<>+0xf0(SB)/8, $0x0807060504030201		1058	DATA shifts<>+0xf0(SB)/8, $0x0807060504030201
	1035	DATA shifts<>+0xf8(SB)/8, $0xff0f0e0d0c0b0a09		1059	DATA shifts<>+0xf8(SB)/8, $0xff0f0e0d0c0b0a09
n	1036	GLOBL shifts<>(SB),RODATA,$256	n	1060	GLOBL shifts<>(SB), RODATA, $256
	1037			1061
n	1038	TEXT runtime·memeq(SB),NOSPLIT,$0-25	n	1062	TEXT runtime·memeq(SB), NOSPLIT, $0-25
	1039	MOVQ a+0(FP), SI		1063	MOVQ a+0(FP), SI
	1040	MOVQ b+8(FP), DI		1064	MOVQ b+8(FP), DI
	1041	MOVQ size+16(FP), BX		1065	MOVQ size+16(FP), BX
	1042	CALL runtime·memeqbody(SB)		1066	CALL runtime·memeqbody(SB)
	1043	MOVB AX, ret+24(FP)		1067	MOVB AX, ret+24(FP)
	1044	RET		1068	RET
	1045			1069
	1046	// eqstring tests whether two strings are equal.		1070	// eqstring tests whether two strings are equal.
	1047	// See runtime_test.go:eqstring_generic for		1071	// See runtime_test.go:eqstring_generic for
	1048	// equivalent Go code.		1072	// equivalent Go code.
n	1049	TEXT runtime·eqstring(SB),NOSPLIT,$0-33	n	1073	TEXT runtime·eqstring(SB), NOSPLIT, $0-33
	1050	MOVQ s1len+8(FP), AX		1074	MOVQ s1len+8(FP), AX
	1051	MOVQ s2len+24(FP), BX		1075	MOVQ s2len+24(FP), BX
	1052	CMPQ AX, BX		1076	CMPQ AX, BX
	1053	JNE different		1077	JNE different
	1054	MOVQ s1str+0(FP), SI		1078	MOVQ s1str+0(FP), SI
	1055	MOVQ s2str+16(FP), DI		1079	MOVQ s2str+16(FP), DI
	1056	CMPQ SI, DI		1080	CMPQ SI, DI
	1057	JEQ same		1081	JEQ same
	1058	CALL runtime·memeqbody(SB)		1082	CALL runtime·memeqbody(SB)
	1059	MOVB AX, v+32(FP)		1083	MOVB AX, v+32(FP)
	1060	RET		1084	RET
n			n	1085
	1061	same:		1086	same:
n	1062	MOVB $1, v+32(FP)	n	1087	MOVB $1, v+32(FP)
	1063	RET		1088	RET
n			n	1089
	1064	different:		1090	different:
n	1065	MOVB $0, v+32(FP)	n	1091	MOVB $0, v+32(FP)
	1066	RET		1092	RET
	1067			1093
	1068	// a in SI		1094	// a in SI
	1069	// b in DI		1095	// b in DI
	1070	// count in BX		1096	// count in BX
n	1071	TEXT runtime·memeqbody(SB),NOSPLIT,$0-0	n	1097	TEXT runtime·memeqbody(SB), NOSPLIT, $0-0
	1072	XORQ AX, AX		1098	XORQ AX, AX
	1073			1099
n	1074	CMPQ BX, $8	n	1100	CMPQ BX, $8
	1075	JB small		1101	JB small
	1076			1102
	1077	// 64 bytes at a time using xmm registers		1103	// 64 bytes at a time using xmm registers
	1078	hugeloop:		1104	hugeloop:
n	1079	CMPQ BX, $64	n	1105	CMPQ BX, $64
	1080	JB bigloop		1106	JB bigloop
	1081	MOVOU (SI), X0		1107	MOVOU (SI), X0
	1082	MOVOU (DI), X1		1108	MOVOU (DI), X1
	1083	MOVOU 16(SI), X2		1109	MOVOU 16(SI), X2
	1084	MOVOU 16(DI), X3		1110	MOVOU 16(DI), X3
	1085	MOVOU 32(SI), X4		1111	MOVOU 32(SI), X4
	1086	MOVOU 32(DI), X5		1112	MOVOU 32(DI), X5
	1087	MOVOU 48(SI), X6		1113	MOVOU 48(SI), X6
	1088	MOVOU 48(DI), X7		1114	MOVOU 48(DI), X7
	1089	PCMPEQB X1, X0		1115	PCMPEQB X1, X0
	1090	PCMPEQB X3, X2		1116	PCMPEQB X3, X2
	1091	PCMPEQB X5, X4		1117	PCMPEQB X5, X4
	1092	PCMPEQB X7, X6		1118	PCMPEQB X7, X6
	1093	PAND X2, X0		1119	PAND X2, X0
	1094	PAND X6, X4		1120	PAND X6, X4
	1095	PAND X4, X0		1121	PAND X4, X0
	1096	PMOVMSKB X0, DX		1122	PMOVMSKB X0, DX
n	1097	ADDQ $64, SI	n	1123	ADDQ $64, SI
	1098	ADDQ $64, DI		1124	ADDQ $64, DI
	1099	SUBQ $64, BX		1125	SUBQ $64, BX
	1100	CMPL DX, $0xffff		1126	CMPL DX, $0xffff
	1101	JEQ hugeloop		1127	JEQ hugeloop
	1102	RET		1128	RET
	1103			1129
n	1104	// 8 bytes at a time using 64-bit register	n	1130	// 8 bytes at a time using 64-bit register
	1105	bigloop:		1131	bigloop:
n	1106	CMPQ BX, $8	n	1132	CMPQ BX, $8
	1107	JBE leftover		1133	JBE leftover
	1108	MOVQ (SI), CX		1134	MOVQ (SI), CX
	1109	MOVQ (DI), DX		1135	MOVQ (DI), DX
	1110	ADDQ $8, SI		1136	ADDQ $8, SI
	1111	ADDQ $8, DI		1137	ADDQ $8, DI
	1112	SUBQ $8, BX		1138	SUBQ $8, BX
	1113	CMPQ CX, DX		1139	CMPQ CX, DX
	1114	JEQ bigloop		1140	JEQ bigloop
	1115	RET		1141	RET
	1116			1142
n	1117	// remaining 0-8 bytes	n	1143	// remaining 0-8 bytes
	1118	leftover:		1144	leftover:
n	1119	MOVQ -8(SI)(BX*1), CX	n	1145	MOVQ -8(SI)(BX*1), CX
	1120	MOVQ -8(DI)(BX*1), DX		1146	MOVQ -8(DI)(BX*1), DX
	1121	CMPQ CX, DX		1147	CMPQ CX, DX
	1122	SETEQ AX		1148	SETEQ AX
	1123	RET		1149	RET
	1124			1150
	1125	small:		1151	small:
n	1126	CMPQ BX, $0	n	1152	CMPQ BX, $0
	1127	JEQ equal		1153	JEQ equal
	1128			1154
n	1129	LEAQ 0(BX*8), CX	n	1155	LEAQ 0(BX*8), CX
	1130	NEGQ CX		1156	NEGQ CX
	1131			1157
n	1132	CMPB SI, $0xf8	n	1158	CMPB SI, $0xf8
	1133	JA si_high		1159	JA si_high
	1134			1160
	1135	// load at SI won't cross a page boundary.		1161	// load at SI won't cross a page boundary.
n	1136	MOVQ (SI), SI	n	1162	MOVQ (SI), SI
	1137	JMP si_finish		1163	JMP si_finish
				1164
	1138	si_high:		1165	si_high:
	1139	// address ends in 11111xxx. Load up to bytes we want, move to correct position.		1166	// address ends in 11111xxx. Load up to bytes we want, move to correct position.
n	1140	MOVQ -8(SI)(BX*1), SI	n	1167	MOVQ -8(SI)(BX*1), SI
	1141	SHRQ CX, SI		1168	SHRQ CX, SI
				1169
	1142	si_finish:		1170	si_finish:
	1143			1171
	1144	// same for DI.		1172	// same for DI.
n	1145	CMPB DI, $0xf8	n	1173	CMPB DI, $0xf8
	1146	JA di_high		1174	JA di_high
	1147	MOVQ (DI), DI		1175	MOVQ (DI), DI
	1148	JMP di_finish		1176	JMP di_finish
				1177
	1149	di_high:		1178	di_high:
n	1150	MOVQ -8(DI)(BX*1), DI	n	1179	MOVQ -8(DI)(BX*1), DI
	1151	SHRQ CX, DI		1180	SHRQ CX, DI
				1181
	1152	di_finish:		1182	di_finish:
	1153			1183
n	1154	SUBQ SI, DI	n	1184	SUBQ SI, DI
	1155	SHLQ CX, DI		1185	SHLQ CX, DI
				1186
	1156	equal:		1187	equal:
n	1157	SETEQ AX	n	1188	SETEQ AX
	1158	RET		1189	RET
	1159			1190
n	1160	TEXT runtime·cmpstring(SB),NOSPLIT,$0-40	n	1191	TEXT runtime·cmpstring(SB), NOSPLIT, $0-40
	1161	MOVQ s1_base+0(FP), SI		1192	MOVQ s1_base+0(FP), SI
	1162	MOVQ s1_len+8(FP), BX		1193	MOVQ s1_len+8(FP), BX
	1163	MOVQ s2_base+16(FP), DI		1194	MOVQ s2_base+16(FP), DI
	1164	MOVQ s2_len+24(FP), DX		1195	MOVQ s2_len+24(FP), DX
	1165	CALL runtime·cmpbody(SB)		1196	CALL runtime·cmpbody(SB)
	1166	MOVQ AX, ret+32(FP)		1197	MOVQ AX, ret+32(FP)
	1167	RET		1198	RET
	1168			1199
n	1169	TEXT runtime·cmpbytes(SB),NOSPLIT,$0-56	n	1200	TEXT runtime·cmpbytes(SB), NOSPLIT, $0-56
	1170	MOVQ s1+0(FP), SI		1201	MOVQ s1+0(FP), SI
	1171	MOVQ s1+8(FP), BX		1202	MOVQ s1+8(FP), BX
	1172	MOVQ s2+24(FP), DI		1203	MOVQ s2+24(FP), DI
	1173	MOVQ s2+32(FP), DX		1204	MOVQ s2+32(FP), DX
	1174	CALL runtime·cmpbody(SB)		1205	CALL runtime·cmpbody(SB)
	1175	MOVQ AX, res+48(FP)		1206	MOVQ AX, res+48(FP)
	1176	RET		1207	RET
	1177			1208
	1178	// input:		1209	// input:
	1179	// SI = a		1210	// SI = a
	1180	// DI = b		1211	// DI = b
	1181	// BX = alen		1212	// BX = alen
	1182	// DX = blen		1213	// DX = blen
	1183	// output:		1214	// output:
	1184	// AX = 1/0/-1		1215	// AX = 1/0/-1
n	1185	TEXT runtime·cmpbody(SB),NOSPLIT,$0-0	n	1216	TEXT runtime·cmpbody(SB), NOSPLIT, $0-0
	1186	CMPQ SI, DI		1217	CMPQ SI, DI
	1187	JEQ cmp_allsame		1218	JEQ cmp_allsame
	1188	CMPQ BX, DX		1219	CMPQ BX, DX
	1189	MOVQ DX, BP		1220	MOVQ DX, BP
	1190	CMOVQLT BX, BP // BP = min(alen, blen) = # of bytes to compare		1221	CMOVQLT BX, BP // BP = min(alen, blen) = # of bytes to compare
	1191	CMPQ BP, $8		1222	CMPQ BP, $8
	1192	JB cmp_small		1223	JB cmp_small
	1193			1224
	1194	cmp_loop:		1225	cmp_loop:
n	1195	CMPQ BP, $16	n	1226	CMPQ BP, $16
	1196	JBE cmp_0through16		1227	JBE cmp_0through16
	1197	MOVOU (SI), X0		1228	MOVOU (SI), X0
	1198	MOVOU (DI), X1		1229	MOVOU (DI), X1
	1199	PCMPEQB X0, X1		1230	PCMPEQB X0, X1
	1200	PMOVMSKB X1, AX		1231	PMOVMSKB X1, AX
n	1201	XORQ $0xffff, AX // convert EQ to NE	n	1232	XORQ $0xffff, AX // convert EQ to NE
	1202	JNE cmp_diff16 // branch if at least one byte is not equal		1233	JNE cmp_diff16 // branch if at least one byte is not equal
	1203	ADDQ $16, SI		1234	ADDQ $16, SI
	1204	ADDQ $16, DI		1235	ADDQ $16, DI
	1205	SUBQ $16, BP		1236	SUBQ $16, BP
	1206	JMP cmp_loop		1237	JMP cmp_loop
	1207			1238
	1208	// AX = bit mask of differences		1239	// AX = bit mask of differences
	1209	cmp_diff16:		1240	cmp_diff16:
n	1210	BSFQ AX, BX // index of first byte that differs	n	1241	BSFQ AX, BX // index of first byte that differs
	1211	XORQ AX, AX		1242	XORQ AX, AX
	1212	MOVB (SI)(BX*1), CX		1243	MOVB (SI)(BX*1), CX
	1213	CMPB CX, (DI)(BX*1)		1244	CMPB CX, (DI)(BX*1)
	1214	SETHI AX		1245	SETHI AX
	1215	LEAQ -1(AX*2), AX // convert 1/0 to +1/-1		1246	LEAQ -1(AX*2), AX // convert 1/0 to +1/-1
	1216	RET		1247	RET
	1217			1248
n	1218	// 0 through 16 bytes left, alen>=8, blen>=8	n	1249	// 0 through 16 bytes left, alen>=8, blen>=8
	1219	cmp_0through16:		1250	cmp_0through16:
n	1220	CMPQ BP, $8	n	1251	CMPQ BP, $8
	1221	JBE cmp_0through8		1252	JBE cmp_0through8
	1222	MOVQ (SI), AX		1253	MOVQ (SI), AX
	1223	MOVQ (DI), CX		1254	MOVQ (DI), CX
	1224	CMPQ AX, CX		1255	CMPQ AX, CX
	1225	JNE cmp_diff8		1256	JNE cmp_diff8
				1257
	1226	cmp_0through8:		1258	cmp_0through8:
n	1227	MOVQ -8(SI)(BP*1), AX	n	1259	MOVQ -8(SI)(BP*1), AX
	1228	MOVQ -8(DI)(BP*1), CX		1260	MOVQ -8(DI)(BP*1), CX
	1229	CMPQ AX, CX		1261	CMPQ AX, CX
	1230	JEQ cmp_allsame		1262	JEQ cmp_allsame
	1231			1263
	1232	// AX and CX contain parts of a and b that differ.		1264	// AX and CX contain parts of a and b that differ.
	1233	cmp_diff8:		1265	cmp_diff8:
n	1234	BSWAPQ AX // reverse order of bytes	n	1266	BSWAPQ AX // reverse order of bytes
	1235	BSWAPQ CX		1267	BSWAPQ CX
	1236	XORQ AX, CX		1268	XORQ AX, CX
	1237	BSRQ CX, CX // index of highest bit difference		1269	BSRQ CX, CX // index of highest bit difference
	1238	SHRQ CX, AX // move a's bit to bottom		1270	SHRQ CX, AX // move a's bit to bottom
	1239	ANDQ $1, AX // mask bit		1271	ANDQ $1, AX // mask bit
	1240	LEAQ -1(AX*2), AX // 1/0 => +1/-1		1272	LEAQ -1(AX*2), AX // 1/0 => +1/-1
	1241	RET		1273	RET
	1242			1274
n	1243	// 0-7 bytes in common	n	1275	// 0-7 bytes in common
	1244	cmp_small:		1276	cmp_small:
n	1245	LEAQ (BP*8), CX // bytes left -> bits left	n	1277	LEAQ (BP*8), CX // bytes left -> bits left
	1246	NEGQ CX // - bits lift (== 64 - bits left mod 64)		1278	NEGQ CX // - bits lift (== 64 - bits left mod 64)
	1247	JEQ cmp_allsame		1279	JEQ cmp_allsame
	1248			1280
	1249	// load bytes of a into high bytes of AX		1281	// load bytes of a into high bytes of AX
n	1250	CMPB SI, $0xf8	n	1282	CMPB SI, $0xf8
	1251	JA cmp_si_high		1283	JA cmp_si_high
	1252	MOVQ (SI), SI		1284	MOVQ (SI), SI
	1253	JMP cmp_si_finish		1285	JMP cmp_si_finish
				1286
	1254	cmp_si_high:		1287	cmp_si_high:
n	1255	MOVQ -8(SI)(BP*1), SI	n	1288	MOVQ -8(SI)(BP*1), SI
	1256	SHRQ CX, SI		1289	SHRQ CX, SI
				1290
	1257	cmp_si_finish:		1291	cmp_si_finish:
n	1258	SHLQ CX, SI	n	1292	SHLQ CX, SI
	1259			1293
	1260	// load bytes of b in to high bytes of BX		1294	// load bytes of b in to high bytes of BX
n	1261	CMPB DI, $0xf8	n	1295	CMPB DI, $0xf8
	1262	JA cmp_di_high		1296	JA cmp_di_high
	1263	MOVQ (DI), DI		1297	MOVQ (DI), DI
	1264	JMP cmp_di_finish		1298	JMP cmp_di_finish
				1299
	1265	cmp_di_high:		1300	cmp_di_high:
n	1266	MOVQ -8(DI)(BP*1), DI	n	1301	MOVQ -8(DI)(BP*1), DI
	1267	SHRQ CX, DI		1302	SHRQ CX, DI
				1303
	1268	cmp_di_finish:		1304	cmp_di_finish:
n	1269	SHLQ CX, DI	n	1305	SHLQ CX, DI
	1270			1306
n	1271	BSWAPQ SI // reverse order of bytes	n	1307	BSWAPQ SI // reverse order of bytes
	1272	BSWAPQ DI		1308	BSWAPQ DI
	1273	XORQ SI, DI // find bit differences		1309	XORQ SI, DI // find bit differences
	1274	JEQ cmp_allsame		1310	JEQ cmp_allsame
	1275	BSRQ DI, CX // index of highest bit difference		1311	BSRQ DI, CX // index of highest bit difference
	1276	SHRQ CX, SI // move a's bit to bottom		1312	SHRQ CX, SI // move a's bit to bottom
	1277	ANDQ $1, SI // mask bit		1313	ANDQ $1, SI // mask bit
	1278	LEAQ -1(SI*2), AX // 1/0 => +1/-1		1314	LEAQ -1(SI*2), AX // 1/0 => +1/-1
	1279	RET		1315	RET
	1280			1316
	1281	cmp_allsame:		1317	cmp_allsame:
n	1282	XORQ AX, AX	n	1318	XORQ AX, AX
	1283	XORQ CX, CX		1319	XORQ CX, CX
	1284	CMPQ BX, DX		1320	CMPQ BX, DX
	1285	SETGT AX // 1 if alen > blen		1321	SETGT AX // 1 if alen > blen
	1286	SETEQ CX // 1 if alen == blen		1322	SETEQ CX // 1 if alen == blen
	1287	LEAQ -1(CX)(AX*2), AX // 1,0,-1 result		1323	LEAQ -1(CX)(AX*2), AX // 1,0,-1 result
	1288	RET		1324	RET
	1289			1325
n	1290	TEXT bytes·IndexByte(SB),NOSPLIT,$0	n	1326	TEXT bytes·IndexByte(SB), NOSPLIT, $0
	1291	MOVQ s+0(FP), SI		1327	MOVQ s+0(FP), SI
	1292	MOVQ s_len+8(FP), BX		1328	MOVQ s_len+8(FP), BX
	1293	MOVB c+24(FP), AL		1329	MOVB c+24(FP), AL
	1294	CALL runtime·indexbytebody(SB)		1330	CALL runtime·indexbytebody(SB)
	1295	MOVQ AX, ret+32(FP)		1331	MOVQ AX, ret+32(FP)
	1296	RET		1332	RET
	1297			1333
n	1298	TEXT strings·IndexByte(SB),NOSPLIT,$0	n	1334	TEXT strings·IndexByte(SB), NOSPLIT, $0
	1299	MOVQ s+0(FP), SI		1335	MOVQ s+0(FP), SI
	1300	MOVQ s_len+8(FP), BX		1336	MOVQ s_len+8(FP), BX
	1301	MOVB c+16(FP), AL		1337	MOVB c+16(FP), AL
	1302	CALL runtime·indexbytebody(SB)		1338	CALL runtime·indexbytebody(SB)
	1303	MOVQ AX, ret+24(FP)		1339	MOVQ AX, ret+24(FP)
	1304	RET		1340	RET
	1305			1341
	1306	// input:		1342	// input:
	1307	// SI: data		1343	// SI: data
	1308	// BX: data len		1344	// BX: data len
	1309	// AL: byte sought		1345	// AL: byte sought
	1310	// output:		1346	// output:
	1311	// AX		1347	// AX
n	1312	TEXT runtime·indexbytebody(SB),NOSPLIT,$0	n	1348	TEXT runtime·indexbytebody(SB), NOSPLIT, $0
	1313	MOVQ SI, DI		1349	MOVQ SI, DI
	1314			1350
	1315	CMPQ BX, $16		1351	CMPQ BX, $16
n	1316	JLT indexbyte_small	n	1352	JLT indexbyte_small
	1317			1353
	1318	// round up to first 16-byte boundary		1354	// round up to first 16-byte boundary
	1319	TESTQ $15, SI		1355	TESTQ $15, SI
n	1320	JZ aligned	n	1356	JZ aligned
	1321	MOVQ SI, CX		1357	MOVQ SI, CX
	1322	ANDQ $~15, CX		1358	ANDQ $~15, CX
	1323	ADDQ $16, CX		1359	ADDQ $16, CX
	1324			1360
	1325	// search the beginning		1361	// search the beginning
n	1326	SUBQ SI, CX	n	1362	SUBQ SI, CX
	1327	REPN; SCASB		1363	REPN; SCASB
n	1328	JZ success	n	1364	JZ success
	1329			1365
n	1330	// DI is 16-byte aligned; get ready to search using SSE instructions	n	1366	// DI is 16-byte aligned; get ready to search using SSE instructions
	1331	aligned:		1367	aligned:
	1332	// round down to last 16-byte boundary		1368	// round down to last 16-byte boundary
	1333	MOVQ BX, R11		1369	MOVQ BX, R11
	1334	ADDQ SI, R11		1370	ADDQ SI, R11
	1335	ANDQ $~15, R11		1371	ANDQ $~15, R11
	1336			1372
	1337	// shuffle X0 around so that each byte contains c		1373	// shuffle X0 around so that each byte contains c
n	1338	MOVD AX, X0	n	1374	MOVD AX, X0
	1339	PUNPCKLBW X0, X0		1375	PUNPCKLBW X0, X0
	1340	PUNPCKLBW X0, X0		1376	PUNPCKLBW X0, X0
n	1341	PSHUFL $0, X0, X0	n	1377	PSHUFL $0, X0, X0
	1342	JMP condition		1378	JMP condition
	1343			1379
	1344	sse:		1380	sse:
	1345	// move the next 16-byte chunk of the buffer into X1		1381	// move the next 16-byte chunk of the buffer into X1
	1346	MOVO (DI), X1		1382	MOVO (DI), X1
n			n	1383
	1347	// compare bytes in X0 to X1		1384	// compare bytes in X0 to X1
	1348	PCMPEQB X0, X1		1385	PCMPEQB X0, X1
n			n	1386
	1349	// take the top bit of each byte in X1 and put the result in DX		1387	// take the top bit of each byte in X1 and put the result in DX
	1350	PMOVMSKB X1, DX		1388	PMOVMSKB X1, DX
n	1351	TESTL DX, DX	n	1389	TESTL DX, DX
	1352	JNZ ssesuccess		1390	JNZ ssesuccess
	1353	ADDQ $16, DI		1391	ADDQ $16, DI
	1354			1392
	1355	condition:		1393	condition:
	1356	CMPQ DI, R11		1394	CMPQ DI, R11
n	1357	JLT sse	n	1395	JLT sse
	1358			1396
	1359	// search the end		1397	// search the end
	1360	MOVQ SI, CX		1398	MOVQ SI, CX
	1361	ADDQ BX, CX		1399	ADDQ BX, CX
	1362	SUBQ R11, CX		1400	SUBQ R11, CX
n			n	1401
	1363	// if CX == 0, the zero flag will be set and we'll end up		1402	// if CX == 0, the zero flag will be set and we'll end up
	1364	// returning a false success		1403	// returning a false success
n	1365	JZ failure	n	1404	JZ failure
	1366	REPN; SCASB		1405	REPN; SCASB
n	1367	JZ success	n	1406	JZ success
	1368			1407
	1369	failure:		1408	failure:
	1370	MOVQ $-1, AX		1409	MOVQ $-1, AX
	1371	RET		1410	RET
	1372			1411
	1373	// handle for lengths < 16		1412	// handle for lengths < 16
	1374	indexbyte_small:		1413	indexbyte_small:
n	1375	MOVQ BX, CX	n	1414	MOVQ BX, CX
	1376	REPN; SCASB		1415	REPN; SCASB
n	1377	JZ success	n	1416	JZ success
	1378	MOVQ $-1, AX		1417	MOVQ $-1, AX
	1379	RET		1418	RET
	1380			1419
	1381	// we've found the chunk containing the byte		1420	// we've found the chunk containing the byte
	1382	// now just figure out which specific byte it is		1421	// now just figure out which specific byte it is
	1383	ssesuccess:		1422	ssesuccess:
	1384	// get the index of the least significant set bit		1423	// get the index of the least significant set bit
	1385	BSFW DX, DX		1424	BSFW DX, DX
	1386	SUBQ SI, DI		1425	SUBQ SI, DI
	1387	ADDQ DI, DX		1426	ADDQ DI, DX
	1388	MOVQ DX, AX		1427	MOVQ DX, AX
	1389	RET		1428	RET
	1390			1429
	1391	success:		1430	success:
	1392	SUBQ SI, DI		1431	SUBQ SI, DI
	1393	SUBL $1, DI		1432	SUBL $1, DI
	1394	MOVQ DI, AX		1433	MOVQ DI, AX
	1395	RET		1434	RET
	1396			1435
n	1397	TEXT bytes·Equal(SB),NOSPLIT,$0-49	n	1436	TEXT bytes·Equal(SB), NOSPLIT, $0-49
	1398	MOVQ a_len+8(FP), BX		1437	MOVQ a_len+8(FP), BX
	1399	MOVQ b_len+32(FP), CX		1438	MOVQ b_len+32(FP), CX
	1400	XORQ AX, AX		1439	XORQ AX, AX
	1401	CMPQ BX, CX		1440	CMPQ BX, CX
	1402	JNE eqret		1441	JNE eqret
	1403	MOVQ a+0(FP), SI		1442	MOVQ a+0(FP), SI
	1404	MOVQ b+24(FP), DI		1443	MOVQ b+24(FP), DI
	1405	CALL runtime·memeqbody(SB)		1444	CALL runtime·memeqbody(SB)
				1445
	1406	eqret:		1446	eqret:
n	1407	MOVB AX, ret+48(FP)	n	1447	MOVB AX, ret+48(FP)
	1408	RET		1448	RET
	1409			1449
	1410	// A Duff's device for zeroing memory.		1450	// A Duff's device for zeroing memory.
	1411	// The compiler jumps to computed addresses within		1451	// The compiler jumps to computed addresses within
	1412	// this routine to zero chunks of memory. Do not		1452	// this routine to zero chunks of memory. Do not
	1413	// change this code without also changing the code		1453	// change this code without also changing the code
	1414	// in ../../cmd/6g/ggen.c:clearfat.		1454	// in ../../cmd/6g/ggen.c:clearfat.
	1415	// AX: zero		1455	// AX: zero
	1416	// DI: ptr to memory to be zeroed		1456	// DI: ptr to memory to be zeroed
	1417	// DI is updated as a side effect.		1457	// DI is updated as a side effect.
	1418	TEXT runtime·duffzero(SB), NOSPLIT, $0-0		1458	TEXT runtime·duffzero(SB), NOSPLIT, $0-0
	1419	STOSQ		1459	STOSQ
	1420	STOSQ		1460	STOSQ
	1421	STOSQ		1461	STOSQ
	1422	STOSQ		1462	STOSQ
	1423	STOSQ		1463	STOSQ
	1424	STOSQ		1464	STOSQ
	1425	STOSQ		1465	STOSQ
	1426	STOSQ		1466	STOSQ
	1427	STOSQ		1467	STOSQ
	1428	STOSQ		1468	STOSQ
	1429	STOSQ		1469	STOSQ
	1430	STOSQ		1470	STOSQ
	1431	STOSQ		1471	STOSQ
	1432	STOSQ		1472	STOSQ
	1433	STOSQ		1473	STOSQ
	1434	STOSQ		1474	STOSQ
	1435	STOSQ		1475	STOSQ
	1436	STOSQ		1476	STOSQ
	1437	STOSQ		1477	STOSQ
	1438	STOSQ		1478	STOSQ
	1439	STOSQ		1479	STOSQ
	1440	STOSQ		1480	STOSQ
	1441	STOSQ		1481	STOSQ
	1442	STOSQ		1482	STOSQ
	1443	STOSQ		1483	STOSQ
	1444	STOSQ		1484	STOSQ
	1445	STOSQ		1485	STOSQ
	1446	STOSQ		1486	STOSQ
	1447	STOSQ		1487	STOSQ
	1448	STOSQ		1488	STOSQ
	1449	STOSQ		1489	STOSQ
	1450	STOSQ		1490	STOSQ
	1451	STOSQ		1491	STOSQ
	1452	STOSQ		1492	STOSQ
	1453	STOSQ		1493	STOSQ
	1454	STOSQ		1494	STOSQ
	1455	STOSQ		1495	STOSQ
	1456	STOSQ		1496	STOSQ
	1457	STOSQ		1497	STOSQ
	1458	STOSQ		1498	STOSQ
	1459	STOSQ		1499	STOSQ
	1460	STOSQ		1500	STOSQ
	1461	STOSQ		1501	STOSQ
	1462	STOSQ		1502	STOSQ
	1463	STOSQ		1503	STOSQ
	1464	STOSQ		1504	STOSQ
	1465	STOSQ		1505	STOSQ
	1466	STOSQ		1506	STOSQ
	1467	STOSQ		1507	STOSQ
	1468	STOSQ		1508	STOSQ
	1469	STOSQ		1509	STOSQ
	1470	STOSQ		1510	STOSQ
	1471	STOSQ		1511	STOSQ
	1472	STOSQ		1512	STOSQ
	1473	STOSQ		1513	STOSQ
	1474	STOSQ		1514	STOSQ
	1475	STOSQ		1515	STOSQ
	1476	STOSQ		1516	STOSQ
	1477	STOSQ		1517	STOSQ
	1478	STOSQ		1518	STOSQ
	1479	STOSQ		1519	STOSQ
	1480	STOSQ		1520	STOSQ
	1481	STOSQ		1521	STOSQ
	1482	STOSQ		1522	STOSQ
	1483	STOSQ		1523	STOSQ
	1484	STOSQ		1524	STOSQ
	1485	STOSQ		1525	STOSQ
	1486	STOSQ		1526	STOSQ
	1487	STOSQ		1527	STOSQ
	1488	STOSQ		1528	STOSQ
	1489	STOSQ		1529	STOSQ
	1490	STOSQ		1530	STOSQ
	1491	STOSQ		1531	STOSQ
	1492	STOSQ		1532	STOSQ
	1493	STOSQ		1533	STOSQ
	1494	STOSQ		1534	STOSQ
	1495	STOSQ		1535	STOSQ
	1496	STOSQ		1536	STOSQ
	1497	STOSQ		1537	STOSQ
	1498	STOSQ		1538	STOSQ
	1499	STOSQ		1539	STOSQ
	1500	STOSQ		1540	STOSQ
	1501	STOSQ		1541	STOSQ
	1502	STOSQ		1542	STOSQ
	1503	STOSQ		1543	STOSQ
	1504	STOSQ		1544	STOSQ
	1505	STOSQ		1545	STOSQ
	1506	STOSQ		1546	STOSQ
	1507	STOSQ		1547	STOSQ
	1508	STOSQ		1548	STOSQ
	1509	STOSQ		1549	STOSQ
	1510	STOSQ		1550	STOSQ
	1511	STOSQ		1551	STOSQ
	1512	STOSQ		1552	STOSQ
	1513	STOSQ		1553	STOSQ
	1514	STOSQ		1554	STOSQ
	1515	STOSQ		1555	STOSQ
	1516	STOSQ		1556	STOSQ
	1517	STOSQ		1557	STOSQ
	1518	STOSQ		1558	STOSQ
	1519	STOSQ		1559	STOSQ
	1520	STOSQ		1560	STOSQ
	1521	STOSQ		1561	STOSQ
	1522	STOSQ		1562	STOSQ
	1523	STOSQ		1563	STOSQ
	1524	STOSQ		1564	STOSQ
	1525	STOSQ		1565	STOSQ
	1526	STOSQ		1566	STOSQ
	1527	STOSQ		1567	STOSQ
	1528	STOSQ		1568	STOSQ
	1529	STOSQ		1569	STOSQ
	1530	STOSQ		1570	STOSQ
	1531	STOSQ		1571	STOSQ
	1532	STOSQ		1572	STOSQ
	1533	STOSQ		1573	STOSQ
	1534	STOSQ		1574	STOSQ
	1535	STOSQ		1575	STOSQ
	1536	STOSQ		1576	STOSQ
	1537	STOSQ		1577	STOSQ
	1538	STOSQ		1578	STOSQ
	1539	STOSQ		1579	STOSQ
	1540	STOSQ		1580	STOSQ
	1541	STOSQ		1581	STOSQ
	1542	STOSQ		1582	STOSQ
	1543	STOSQ		1583	STOSQ
	1544	STOSQ		1584	STOSQ
	1545	STOSQ		1585	STOSQ
	1546	STOSQ		1586	STOSQ
	1547	RET		1587	RET
	1548			1588
	1549	// A Duff's device for copying memory.		1589	// A Duff's device for copying memory.
	1550	// The compiler jumps to computed addresses within		1590	// The compiler jumps to computed addresses within
	1551	// this routine to copy chunks of memory. Source		1591	// this routine to copy chunks of memory. Source
	1552	// and destination must not overlap. Do not		1592	// and destination must not overlap. Do not
	1553	// change this code without also changing the code		1593	// change this code without also changing the code
	1554	// in ../../cmd/6g/cgen.c:sgen.		1594	// in ../../cmd/6g/cgen.c:sgen.
	1555	// SI: ptr to source memory		1595	// SI: ptr to source memory
	1556	// DI: ptr to destination memory		1596	// DI: ptr to destination memory
	1557	// SI and DI are updated as a side effect.		1597	// SI and DI are updated as a side effect.
	1558			1598
	1559	// NOTE: this is equivalent to a sequence of MOVSQ but		1599	// NOTE: this is equivalent to a sequence of MOVSQ but
	1560	// for some reason that is 3.5x slower than this code.		1600	// for some reason that is 3.5x slower than this code.
	1561	// The STOSQ above seem fine, though.		1601	// The STOSQ above seem fine, though.
	1562	TEXT runtime·duffcopy(SB), NOSPLIT, $0-0		1602	TEXT runtime·duffcopy(SB), NOSPLIT, $0-0
n	1563	MOVQ (SI),CX	n	1603	MOVQ (SI), CX
	1564	ADDQ $8,SI		1604	ADDQ $8, SI
	1565	MOVQ CX,(DI)		1605	MOVQ CX, (DI)
	1566	ADDQ $8,DI		1606	ADDQ $8, DI
	1567			1607
n	1568	MOVQ (SI),CX	n	1608	MOVQ (SI), CX
	1569	ADDQ $8,SI		1609	ADDQ $8, SI
	1570	MOVQ CX,(DI)		1610	MOVQ CX, (DI)
	1571	ADDQ $8,DI		1611	ADDQ $8, DI
	1572			1612
n	1573	MOVQ (SI),CX	n	1613	MOVQ (SI), CX
	1574	ADDQ $8,SI		1614	ADDQ $8, SI
	1575	MOVQ CX,(DI)		1615	MOVQ CX, (DI)
	1576	ADDQ $8,DI		1616	ADDQ $8, DI
	1577			1617
n	1578	MOVQ (SI),CX	n	1618	MOVQ (SI), CX
	1579	ADDQ $8,SI		1619	ADDQ $8, SI
	1580	MOVQ CX,(DI)		1620	MOVQ CX, (DI)
	1581	ADDQ $8,DI		1621	ADDQ $8, DI
	1582			1622
n	1583	MOVQ (SI),CX	n	1623	MOVQ (SI), CX
	1584	ADDQ $8,SI		1624	ADDQ $8, SI
	1585	MOVQ CX,(DI)		1625	MOVQ CX, (DI)
	1586	ADDQ $8,DI		1626	ADDQ $8, DI
	1587			1627
n	1588	MOVQ (SI),CX	n	1628	MOVQ (SI), CX
	1589	ADDQ $8,SI		1629	ADDQ $8, SI
	1590	MOVQ CX,(DI)		1630	MOVQ CX, (DI)
	1591	ADDQ $8,DI		1631	ADDQ $8, DI
	1592			1632
n	1593	MOVQ (SI),CX	n	1633	MOVQ (SI), CX
	1594	ADDQ $8,SI		1634	ADDQ $8, SI
	1595	MOVQ CX,(DI)		1635	MOVQ CX, (DI)
	1596	ADDQ $8,DI		1636	ADDQ $8, DI
	1597			1637
n	1598	MOVQ (SI),CX	n	1638	MOVQ (SI), CX
	1599	ADDQ $8,SI		1639	ADDQ $8, SI
	1600	MOVQ CX,(DI)		1640	MOVQ CX, (DI)
	1601	ADDQ $8,DI		1641	ADDQ $8, DI
	1602			1642
n	1603	MOVQ (SI),CX	n	1643	MOVQ (SI), CX
	1604	ADDQ $8,SI		1644	ADDQ $8, SI
	1605	MOVQ CX,(DI)		1645	MOVQ CX, (DI)
	1606	ADDQ $8,DI		1646	ADDQ $8, DI
	1607			1647
n	1608	MOVQ (SI),CX	n	1648	MOVQ (SI), CX
	1609	ADDQ $8,SI		1649	ADDQ $8, SI
	1610	MOVQ CX,(DI)		1650	MOVQ CX, (DI)
	1611	ADDQ $8,DI		1651	ADDQ $8, DI
	1612			1652
n	1613	MOVQ (SI),CX	n	1653	MOVQ (SI), CX
	1614	ADDQ $8,SI		1654	ADDQ $8, SI
	1615	MOVQ CX,(DI)		1655	MOVQ CX, (DI)
	1616	ADDQ $8,DI		1656	ADDQ $8, DI
	1617			1657
n	1618	MOVQ (SI),CX	n	1658	MOVQ (SI), CX
	1619	ADDQ $8,SI		1659	ADDQ $8, SI
	1620	MOVQ CX,(DI)		1660	MOVQ CX, (DI)
	1621	ADDQ $8,DI		1661	ADDQ $8, DI
	1622			1662
n	1623	MOVQ (SI),CX	n	1663	MOVQ (SI), CX
	1624	ADDQ $8,SI		1664	ADDQ $8, SI
	1625	MOVQ CX,(DI)		1665	MOVQ CX, (DI)
	1626	ADDQ $8,DI		1666	ADDQ $8, DI
	1627			1667
n	1628	MOVQ (SI),CX	n	1668	MOVQ (SI), CX
	1629	ADDQ $8,SI		1669	ADDQ $8, SI
	1630	MOVQ CX,(DI)		1670	MOVQ CX, (DI)
	1631	ADDQ $8,DI		1671	ADDQ $8, DI
	1632			1672
n	1633	MOVQ (SI),CX	n	1673	MOVQ (SI), CX
	1634	ADDQ $8,SI		1674	ADDQ $8, SI
	1635	MOVQ CX,(DI)		1675	MOVQ CX, (DI)
	1636	ADDQ $8,DI		1676	ADDQ $8, DI
	1637			1677
n	1638	MOVQ (SI),CX	n	1678	MOVQ (SI), CX
	1639	ADDQ $8,SI		1679	ADDQ $8, SI
	1640	MOVQ CX,(DI)		1680	MOVQ CX, (DI)
	1641	ADDQ $8,DI		1681	ADDQ $8, DI
	1642			1682
n	1643	MOVQ (SI),CX	n	1683	MOVQ (SI), CX
	1644	ADDQ $8,SI		1684	ADDQ $8, SI
	1645	MOVQ CX,(DI)		1685	MOVQ CX, (DI)
	1646	ADDQ $8,DI		1686	ADDQ $8, DI
	1647			1687
n	1648	MOVQ (SI),CX	n	1688	MOVQ (SI), CX
	1649	ADDQ $8,SI		1689	ADDQ $8, SI
	1650	MOVQ CX,(DI)		1690	MOVQ CX, (DI)
	1651	ADDQ $8,DI		1691	ADDQ $8, DI
	1652			1692
n	1653	MOVQ (SI),CX	n	1693	MOVQ (SI), CX
	1654	ADDQ $8,SI		1694	ADDQ $8, SI
	1655	MOVQ CX,(DI)		1695	MOVQ CX, (DI)
	1656	ADDQ $8,DI		1696	ADDQ $8, DI
	1657			1697
n	1658	MOVQ (SI),CX	n	1698	MOVQ (SI), CX
	1659	ADDQ $8,SI		1699	ADDQ $8, SI
	1660	MOVQ CX,(DI)		1700	MOVQ CX, (DI)
	1661	ADDQ $8,DI		1701	ADDQ $8, DI
	1662			1702
n	1663	MOVQ (SI),CX	n	1703	MOVQ (SI), CX
	1664	ADDQ $8,SI		1704	ADDQ $8, SI
	1665	MOVQ CX,(DI)		1705	MOVQ CX, (DI)
	1666	ADDQ $8,DI		1706	ADDQ $8, DI
	1667			1707
n	1668	MOVQ (SI),CX	n	1708	MOVQ (SI), CX
	1669	ADDQ $8,SI		1709	ADDQ $8, SI
	1670	MOVQ CX,(DI)		1710	MOVQ CX, (DI)
	1671	ADDQ $8,DI		1711	ADDQ $8, DI
	1672			1712
n	1673	MOVQ (SI),CX	n	1713	MOVQ (SI), CX
	1674	ADDQ $8,SI		1714	ADDQ $8, SI
	1675	MOVQ CX,(DI)		1715	MOVQ CX, (DI)
	1676	ADDQ $8,DI		1716	ADDQ $8, DI
	1677			1717
n	1678	MOVQ (SI),CX	n	1718	MOVQ (SI), CX
	1679	ADDQ $8,SI		1719	ADDQ $8, SI
	1680	MOVQ CX,(DI)		1720	MOVQ CX, (DI)
	1681	ADDQ $8,DI		1721	ADDQ $8, DI
	1682			1722
n	1683	MOVQ (SI),CX	n	1723	MOVQ (SI), CX
	1684	ADDQ $8,SI		1724	ADDQ $8, SI
	1685	MOVQ CX,(DI)		1725	MOVQ CX, (DI)
	1686	ADDQ $8,DI		1726	ADDQ $8, DI
	1687			1727
n	1688	MOVQ (SI),CX	n	1728	MOVQ (SI), CX
	1689	ADDQ $8,SI		1729	ADDQ $8, SI
	1690	MOVQ CX,(DI)		1730	MOVQ CX, (DI)
	1691	ADDQ $8,DI		1731	ADDQ $8, DI
	1692			1732
n	1693	MOVQ (SI),CX	n	1733	MOVQ (SI), CX
	1694	ADDQ $8,SI		1734	ADDQ $8, SI
	1695	MOVQ CX,(DI)		1735	MOVQ CX, (DI)
	1696	ADDQ $8,DI		1736	ADDQ $8, DI
	1697			1737
n	1698	MOVQ (SI),CX	n	1738	MOVQ (SI), CX
	1699	ADDQ $8,SI		1739	ADDQ $8, SI
	1700	MOVQ CX,(DI)		1740	MOVQ CX, (DI)
	1701	ADDQ $8,DI		1741	ADDQ $8, DI
	1702			1742
n	1703	MOVQ (SI),CX	n	1743	MOVQ (SI), CX
	1704	ADDQ $8,SI		1744	ADDQ $8, SI
	1705	MOVQ CX,(DI)		1745	MOVQ CX, (DI)
	1706	ADDQ $8,DI		1746	ADDQ $8, DI
	1707			1747
n	1708	MOVQ (SI),CX	n	1748	MOVQ (SI), CX
	1709	ADDQ $8,SI		1749	ADDQ $8, SI
	1710	MOVQ CX,(DI)		1750	MOVQ CX, (DI)
	1711	ADDQ $8,DI		1751	ADDQ $8, DI
	1712			1752
n	1713	MOVQ (SI),CX	n	1753	MOVQ (SI), CX
	1714	ADDQ $8,SI		1754	ADDQ $8, SI
	1715	MOVQ CX,(DI)		1755	MOVQ CX, (DI)
	1716	ADDQ $8,DI		1756	ADDQ $8, DI
	1717			1757
n	1718	MOVQ (SI),CX	n	1758	MOVQ (SI), CX
	1719	ADDQ $8,SI		1759	ADDQ $8, SI
	1720	MOVQ CX,(DI)		1760	MOVQ CX, (DI)
	1721	ADDQ $8,DI		1761	ADDQ $8, DI
	1722			1762
n	1723	MOVQ (SI),CX	n	1763	MOVQ (SI), CX
	1724	ADDQ $8,SI		1764	ADDQ $8, SI
	1725	MOVQ CX,(DI)		1765	MOVQ CX, (DI)
	1726	ADDQ $8,DI		1766	ADDQ $8, DI
	1727			1767
n	1728	MOVQ (SI),CX	n	1768	MOVQ (SI), CX
	1729	ADDQ $8,SI		1769	ADDQ $8, SI
	1730	MOVQ CX,(DI)		1770	MOVQ CX, (DI)
	1731	ADDQ $8,DI		1771	ADDQ $8, DI
	1732			1772
n	1733	MOVQ (SI),CX	n	1773	MOVQ (SI), CX
	1734	ADDQ $8,SI		1774	ADDQ $8, SI
	1735	MOVQ CX,(DI)		1775	MOVQ CX, (DI)
	1736	ADDQ $8,DI		1776	ADDQ $8, DI
	1737			1777
n	1738	MOVQ (SI),CX	n	1778	MOVQ (SI), CX
	1739	ADDQ $8,SI		1779	ADDQ $8, SI
	1740	MOVQ CX,(DI)		1780	MOVQ CX, (DI)
	1741	ADDQ $8,DI		1781	ADDQ $8, DI
	1742			1782
n	1743	MOVQ (SI),CX	n	1783	MOVQ (SI), CX
	1744	ADDQ $8,SI		1784	ADDQ $8, SI
	1745	MOVQ CX,(DI)		1785	MOVQ CX, (DI)
	1746	ADDQ $8,DI		1786	ADDQ $8, DI
	1747			1787
n	1748	MOVQ (SI),CX	n	1788	MOVQ (SI), CX
	1749	ADDQ $8,SI		1789	ADDQ $8, SI
	1750	MOVQ CX,(DI)		1790	MOVQ CX, (DI)
	1751	ADDQ $8,DI		1791	ADDQ $8, DI
	1752			1792
n	1753	MOVQ (SI),CX	n	1793	MOVQ (SI), CX
	1754	ADDQ $8,SI		1794	ADDQ $8, SI
	1755	MOVQ CX,(DI)		1795	MOVQ CX, (DI)
	1756	ADDQ $8,DI		1796	ADDQ $8, DI
	1757			1797
n	1758	MOVQ (SI),CX	n	1798	MOVQ (SI), CX
	1759	ADDQ $8,SI		1799	ADDQ $8, SI
	1760	MOVQ CX,(DI)		1800	MOVQ CX, (DI)
	1761	ADDQ $8,DI		1801	ADDQ $8, DI
	1762			1802
n	1763	MOVQ (SI),CX	n	1803	MOVQ (SI), CX
	1764	ADDQ $8,SI		1804	ADDQ $8, SI
	1765	MOVQ CX,(DI)		1805	MOVQ CX, (DI)
	1766	ADDQ $8,DI		1806	ADDQ $8, DI
	1767			1807
n	1768	MOVQ (SI),CX	n	1808	MOVQ (SI), CX
	1769	ADDQ $8,SI		1809	ADDQ $8, SI
	1770	MOVQ CX,(DI)		1810	MOVQ CX, (DI)
	1771	ADDQ $8,DI		1811	ADDQ $8, DI
	1772			1812
n	1773	MOVQ (SI),CX	n	1813	MOVQ (SI), CX
	1774	ADDQ $8,SI		1814	ADDQ $8, SI
	1775	MOVQ CX,(DI)		1815	MOVQ CX, (DI)
	1776	ADDQ $8,DI		1816	ADDQ $8, DI
	1777			1817
n	1778	MOVQ (SI),CX	n	1818	MOVQ (SI), CX
	1779	ADDQ $8,SI		1819	ADDQ $8, SI
	1780	MOVQ CX,(DI)		1820	MOVQ CX, (DI)
	1781	ADDQ $8,DI		1821	ADDQ $8, DI
	1782			1822
n	1783	MOVQ (SI),CX	n	1823	MOVQ (SI), CX
	1784	ADDQ $8,SI		1824	ADDQ $8, SI
	1785	MOVQ CX,(DI)		1825	MOVQ CX, (DI)
	1786	ADDQ $8,DI		1826	ADDQ $8, DI
	1787			1827
n	1788	MOVQ (SI),CX	n	1828	MOVQ (SI), CX
	1789	ADDQ $8,SI		1829	ADDQ $8, SI
	1790	MOVQ CX,(DI)		1830	MOVQ CX, (DI)
	1791	ADDQ $8,DI		1831	ADDQ $8, DI
	1792			1832
n	1793	MOVQ (SI),CX	n	1833	MOVQ (SI), CX
	1794	ADDQ $8,SI		1834	ADDQ $8, SI
	1795	MOVQ CX,(DI)		1835	MOVQ CX, (DI)
	1796	ADDQ $8,DI		1836	ADDQ $8, DI
	1797			1837
n	1798	MOVQ (SI),CX	n	1838	MOVQ (SI), CX
	1799	ADDQ $8,SI		1839	ADDQ $8, SI
	1800	MOVQ CX,(DI)		1840	MOVQ CX, (DI)
	1801	ADDQ $8,DI		1841	ADDQ $8, DI
	1802			1842
n	1803	MOVQ (SI),CX	n	1843	MOVQ (SI), CX
	1804	ADDQ $8,SI		1844	ADDQ $8, SI
	1805	MOVQ CX,(DI)		1845	MOVQ CX, (DI)
	1806	ADDQ $8,DI		1846	ADDQ $8, DI
	1807			1847
n	1808	MOVQ (SI),CX	n	1848	MOVQ (SI), CX
	1809	ADDQ $8,SI		1849	ADDQ $8, SI
	1810	MOVQ CX,(DI)		1850	MOVQ CX, (DI)
	1811	ADDQ $8,DI		1851	ADDQ $8, DI
	1812			1852
n	1813	MOVQ (SI),CX	n	1853	MOVQ (SI), CX
	1814	ADDQ $8,SI		1854	ADDQ $8, SI
	1815	MOVQ CX,(DI)		1855	MOVQ CX, (DI)
	1816	ADDQ $8,DI		1856	ADDQ $8, DI
	1817			1857
n	1818	MOVQ (SI),CX	n	1858	MOVQ (SI), CX
	1819	ADDQ $8,SI		1859	ADDQ $8, SI
	1820	MOVQ CX,(DI)		1860	MOVQ CX, (DI)
	1821	ADDQ $8,DI		1861	ADDQ $8, DI
	1822			1862
n	1823	MOVQ (SI),CX	n	1863	MOVQ (SI), CX
	1824	ADDQ $8,SI		1864	ADDQ $8, SI
	1825	MOVQ CX,(DI)		1865	MOVQ CX, (DI)
	1826	ADDQ $8,DI		1866	ADDQ $8, DI
	1827			1867
n	1828	MOVQ (SI),CX	n	1868	MOVQ (SI), CX
	1829	ADDQ $8,SI		1869	ADDQ $8, SI
	1830	MOVQ CX,(DI)		1870	MOVQ CX, (DI)
	1831	ADDQ $8,DI		1871	ADDQ $8, DI
	1832			1872
n	1833	MOVQ (SI),CX	n	1873	MOVQ (SI), CX
	1834	ADDQ $8,SI		1874	ADDQ $8, SI
	1835	MOVQ CX,(DI)		1875	MOVQ CX, (DI)
	1836	ADDQ $8,DI		1876	ADDQ $8, DI
	1837			1877
n	1838	MOVQ (SI),CX	n	1878	MOVQ (SI), CX
	1839	ADDQ $8,SI		1879	ADDQ $8, SI
	1840	MOVQ CX,(DI)		1880	MOVQ CX, (DI)
	1841	ADDQ $8,DI		1881	ADDQ $8, DI
	1842			1882
n	1843	MOVQ (SI),CX	n	1883	MOVQ (SI), CX
	1844	ADDQ $8,SI		1884	ADDQ $8, SI
	1845	MOVQ CX,(DI)		1885	MOVQ CX, (DI)
	1846	ADDQ $8,DI		1886	ADDQ $8, DI
	1847			1887
n	1848	MOVQ (SI),CX	n	1888	MOVQ (SI), CX
	1849	ADDQ $8,SI		1889	ADDQ $8, SI
	1850	MOVQ CX,(DI)		1890	MOVQ CX, (DI)
	1851	ADDQ $8,DI		1891	ADDQ $8, DI
	1852			1892
n	1853	MOVQ (SI),CX	n	1893	MOVQ (SI), CX
	1854	ADDQ $8,SI		1894	ADDQ $8, SI
	1855	MOVQ CX,(DI)		1895	MOVQ CX, (DI)
	1856	ADDQ $8,DI		1896	ADDQ $8, DI
	1857			1897
n	1858	MOVQ (SI),CX	n	1898	MOVQ (SI), CX
	1859	ADDQ $8,SI		1899	ADDQ $8, SI
	1860	MOVQ CX,(DI)		1900	MOVQ CX, (DI)
	1861	ADDQ $8,DI		1901	ADDQ $8, DI
	1862			1902
n	1863	MOVQ (SI),CX	n	1903	MOVQ (SI), CX
	1864	ADDQ $8,SI		1904	ADDQ $8, SI
	1865	MOVQ CX,(DI)		1905	MOVQ CX, (DI)
	1866	ADDQ $8,DI		1906	ADDQ $8, DI
	1867			1907
n	1868	MOVQ (SI),CX	n	1908	MOVQ (SI), CX
	1869	ADDQ $8,SI		1909	ADDQ $8, SI
	1870	MOVQ CX,(DI)		1910	MOVQ CX, (DI)
	1871	ADDQ $8,DI		1911	ADDQ $8, DI
	1872			1912
n	1873	MOVQ (SI),CX	n	1913	MOVQ (SI), CX
	1874	ADDQ $8,SI		1914	ADDQ $8, SI
	1875	MOVQ CX,(DI)		1915	MOVQ CX, (DI)
	1876	ADDQ $8,DI		1916	ADDQ $8, DI
	1877			1917
n	1878	MOVQ (SI),CX	n	1918	MOVQ (SI), CX
	1879	ADDQ $8,SI		1919	ADDQ $8, SI
	1880	MOVQ CX,(DI)		1920	MOVQ CX, (DI)
	1881	ADDQ $8,DI		1921	ADDQ $8, DI
	1882			1922
n	1883	MOVQ (SI),CX	n	1923	MOVQ (SI), CX
	1884	ADDQ $8,SI		1924	ADDQ $8, SI
	1885	MOVQ CX,(DI)		1925	MOVQ CX, (DI)
	1886	ADDQ $8,DI		1926	ADDQ $8, DI
	1887			1927
n	1888	MOVQ (SI),CX	n	1928	MOVQ (SI), CX
	1889	ADDQ $8,SI		1929	ADDQ $8, SI
	1890	MOVQ CX,(DI)		1930	MOVQ CX, (DI)
	1891	ADDQ $8,DI		1931	ADDQ $8, DI
	1892			1932
n	1893	MOVQ (SI),CX	n	1933	MOVQ (SI), CX
	1894	ADDQ $8,SI		1934	ADDQ $8, SI
	1895	MOVQ CX,(DI)		1935	MOVQ CX, (DI)
	1896	ADDQ $8,DI		1936	ADDQ $8, DI
	1897			1937
n	1898	MOVQ (SI),CX	n	1938	MOVQ (SI), CX
	1899	ADDQ $8,SI		1939	ADDQ $8, SI
	1900	MOVQ CX,(DI)		1940	MOVQ CX, (DI)
	1901	ADDQ $8,DI		1941	ADDQ $8, DI
	1902			1942
n	1903	MOVQ (SI),CX	n	1943	MOVQ (SI), CX
	1904	ADDQ $8,SI		1944	ADDQ $8, SI
	1905	MOVQ CX,(DI)		1945	MOVQ CX, (DI)
	1906	ADDQ $8,DI		1946	ADDQ $8, DI
	1907			1947
n	1908	MOVQ (SI),CX	n	1948	MOVQ (SI), CX
	1909	ADDQ $8,SI		1949	ADDQ $8, SI
	1910	MOVQ CX,(DI)		1950	MOVQ CX, (DI)
	1911	ADDQ $8,DI		1951	ADDQ $8, DI
	1912			1952
n	1913	MOVQ (SI),CX	n	1953	MOVQ (SI), CX
	1914	ADDQ $8,SI		1954	ADDQ $8, SI
	1915	MOVQ CX,(DI)		1955	MOVQ CX, (DI)
	1916	ADDQ $8,DI		1956	ADDQ $8, DI
	1917			1957
n	1918	MOVQ (SI),CX	n	1958	MOVQ (SI), CX
	1919	ADDQ $8,SI		1959	ADDQ $8, SI
	1920	MOVQ CX,(DI)		1960	MOVQ CX, (DI)
	1921	ADDQ $8,DI		1961	ADDQ $8, DI
	1922			1962
n	1923	MOVQ (SI),CX	n	1963	MOVQ (SI), CX
	1924	ADDQ $8,SI		1964	ADDQ $8, SI
	1925	MOVQ CX,(DI)		1965	MOVQ CX, (DI)
	1926	ADDQ $8,DI		1966	ADDQ $8, DI
	1927			1967
n	1928	MOVQ (SI),CX	n	1968	MOVQ (SI), CX
	1929	ADDQ $8,SI		1969	ADDQ $8, SI
	1930	MOVQ CX,(DI)		1970	MOVQ CX, (DI)
	1931	ADDQ $8,DI		1971	ADDQ $8, DI
	1932			1972
n	1933	MOVQ (SI),CX	n	1973	MOVQ (SI), CX
	1934	ADDQ $8,SI		1974	ADDQ $8, SI
	1935	MOVQ CX,(DI)		1975	MOVQ CX, (DI)
	1936	ADDQ $8,DI		1976	ADDQ $8, DI
	1937			1977
n	1938	MOVQ (SI),CX	n	1978	MOVQ (SI), CX
	1939	ADDQ $8,SI		1979	ADDQ $8, SI
	1940	MOVQ CX,(DI)		1980	MOVQ CX, (DI)
	1941	ADDQ $8,DI		1981	ADDQ $8, DI
	1942			1982
n	1943	MOVQ (SI),CX	n	1983	MOVQ (SI), CX
	1944	ADDQ $8,SI		1984	ADDQ $8, SI
	1945	MOVQ CX,(DI)		1985	MOVQ CX, (DI)
	1946	ADDQ $8,DI		1986	ADDQ $8, DI
	1947			1987
n	1948	MOVQ (SI),CX	n	1988	MOVQ (SI), CX
	1949	ADDQ $8,SI		1989	ADDQ $8, SI
	1950	MOVQ CX,(DI)		1990	MOVQ CX, (DI)
	1951	ADDQ $8,DI		1991	ADDQ $8, DI
	1952			1992
n	1953	MOVQ (SI),CX	n	1993	MOVQ (SI), CX
	1954	ADDQ $8,SI		1994	ADDQ $8, SI
	1955	MOVQ CX,(DI)		1995	MOVQ CX, (DI)
	1956	ADDQ $8,DI		1996	ADDQ $8, DI
	1957			1997
n	1958	MOVQ (SI),CX	n	1998	MOVQ (SI), CX
	1959	ADDQ $8,SI		1999	ADDQ $8, SI
	1960	MOVQ CX,(DI)		2000	MOVQ CX, (DI)
	1961	ADDQ $8,DI		2001	ADDQ $8, DI
	1962			2002
n	1963	MOVQ (SI),CX	n	2003	MOVQ (SI), CX
	1964	ADDQ $8,SI		2004	ADDQ $8, SI
	1965	MOVQ CX,(DI)		2005	MOVQ CX, (DI)
	1966	ADDQ $8,DI		2006	ADDQ $8, DI
	1967			2007
n	1968	MOVQ (SI),CX	n	2008	MOVQ (SI), CX
	1969	ADDQ $8,SI		2009	ADDQ $8, SI
	1970	MOVQ CX,(DI)		2010	MOVQ CX, (DI)
	1971	ADDQ $8,DI		2011	ADDQ $8, DI
	1972			2012
n	1973	MOVQ (SI),CX	n	2013	MOVQ (SI), CX
	1974	ADDQ $8,SI		2014	ADDQ $8, SI
	1975	MOVQ CX,(DI)		2015	MOVQ CX, (DI)
	1976	ADDQ $8,DI		2016	ADDQ $8, DI
	1977			2017
n	1978	MOVQ (SI),CX	n	2018	MOVQ (SI), CX
	1979	ADDQ $8,SI		2019	ADDQ $8, SI
	1980	MOVQ CX,(DI)		2020	MOVQ CX, (DI)
	1981	ADDQ $8,DI		2021	ADDQ $8, DI
	1982			2022
n	1983	MOVQ (SI),CX	n	2023	MOVQ (SI), CX
	1984	ADDQ $8,SI		2024	ADDQ $8, SI
	1985	MOVQ CX,(DI)		2025	MOVQ CX, (DI)
	1986	ADDQ $8,DI		2026	ADDQ $8, DI
	1987			2027
n	1988	MOVQ (SI),CX	n	2028	MOVQ (SI), CX
	1989	ADDQ $8,SI		2029	ADDQ $8, SI
	1990	MOVQ CX,(DI)		2030	MOVQ CX, (DI)
	1991	ADDQ $8,DI		2031	ADDQ $8, DI
	1992			2032
n	1993	MOVQ (SI),CX	n	2033	MOVQ (SI), CX
	1994	ADDQ $8,SI		2034	ADDQ $8, SI
	1995	MOVQ CX,(DI)		2035	MOVQ CX, (DI)
	1996	ADDQ $8,DI		2036	ADDQ $8, DI
	1997			2037
n	1998	MOVQ (SI),CX	n	2038	MOVQ (SI), CX
	1999	ADDQ $8,SI		2039	ADDQ $8, SI
	2000	MOVQ CX,(DI)		2040	MOVQ CX, (DI)
	2001	ADDQ $8,DI		2041	ADDQ $8, DI
	2002			2042
n	2003	MOVQ (SI),CX	n	2043	MOVQ (SI), CX
	2004	ADDQ $8,SI		2044	ADDQ $8, SI
	2005	MOVQ CX,(DI)		2045	MOVQ CX, (DI)
	2006	ADDQ $8,DI		2046	ADDQ $8, DI
	2007			2047
n	2008	MOVQ (SI),CX	n	2048	MOVQ (SI), CX
	2009	ADDQ $8,SI		2049	ADDQ $8, SI
	2010	MOVQ CX,(DI)		2050	MOVQ CX, (DI)
	2011	ADDQ $8,DI		2051	ADDQ $8, DI
	2012			2052
n	2013	MOVQ (SI),CX	n	2053	MOVQ (SI), CX
	2014	ADDQ $8,SI		2054	ADDQ $8, SI
	2015	MOVQ CX,(DI)		2055	MOVQ CX, (DI)
	2016	ADDQ $8,DI		2056	ADDQ $8, DI
	2017			2057
n	2018	MOVQ (SI),CX	n	2058	MOVQ (SI), CX
	2019	ADDQ $8,SI		2059	ADDQ $8, SI
	2020	MOVQ CX,(DI)		2060	MOVQ CX, (DI)
	2021	ADDQ $8,DI		2061	ADDQ $8, DI
	2022			2062
n	2023	MOVQ (SI),CX	n	2063	MOVQ (SI), CX
	2024	ADDQ $8,SI		2064	ADDQ $8, SI
	2025	MOVQ CX,(DI)		2065	MOVQ CX, (DI)
	2026	ADDQ $8,DI		2066	ADDQ $8, DI
	2027			2067
n	2028	MOVQ (SI),CX	n	2068	MOVQ (SI), CX
	2029	ADDQ $8,SI		2069	ADDQ $8, SI
	2030	MOVQ CX,(DI)		2070	MOVQ CX, (DI)
	2031	ADDQ $8,DI		2071	ADDQ $8, DI
	2032			2072
n	2033	MOVQ (SI),CX	n	2073	MOVQ (SI), CX
	2034	ADDQ $8,SI		2074	ADDQ $8, SI
	2035	MOVQ CX,(DI)		2075	MOVQ CX, (DI)
	2036	ADDQ $8,DI		2076	ADDQ $8, DI
	2037			2077
n	2038	MOVQ (SI),CX	n	2078	MOVQ (SI), CX
	2039	ADDQ $8,SI		2079	ADDQ $8, SI
	2040	MOVQ CX,(DI)		2080	MOVQ CX, (DI)
	2041	ADDQ $8,DI		2081	ADDQ $8, DI
	2042			2082
n	2043	MOVQ (SI),CX	n	2083	MOVQ (SI), CX
	2044	ADDQ $8,SI		2084	ADDQ $8, SI
	2045	MOVQ CX,(DI)		2085	MOVQ CX, (DI)
	2046	ADDQ $8,DI		2086	ADDQ $8, DI
	2047			2087
n	2048	MOVQ (SI),CX	n	2088	MOVQ (SI), CX
	2049	ADDQ $8,SI		2089	ADDQ $8, SI
	2050	MOVQ CX,(DI)		2090	MOVQ CX, (DI)
	2051	ADDQ $8,DI		2091	ADDQ $8, DI
	2052			2092
n	2053	MOVQ (SI),CX	n	2093	MOVQ (SI), CX
	2054	ADDQ $8,SI		2094	ADDQ $8, SI
	2055	MOVQ CX,(DI)		2095	MOVQ CX, (DI)
	2056	ADDQ $8,DI		2096	ADDQ $8, DI
	2057			2097
n	2058	MOVQ (SI),CX	n	2098	MOVQ (SI), CX
	2059	ADDQ $8,SI		2099	ADDQ $8, SI
	2060	MOVQ CX,(DI)		2100	MOVQ CX, (DI)
	2061	ADDQ $8,DI		2101	ADDQ $8, DI
	2062			2102
n	2063	MOVQ (SI),CX	n	2103	MOVQ (SI), CX
	2064	ADDQ $8,SI		2104	ADDQ $8, SI
	2065	MOVQ CX,(DI)		2105	MOVQ CX, (DI)
	2066	ADDQ $8,DI		2106	ADDQ $8, DI
	2067			2107
n	2068	MOVQ (SI),CX	n	2108	MOVQ (SI), CX
	2069	ADDQ $8,SI		2109	ADDQ $8, SI
	2070	MOVQ CX,(DI)		2110	MOVQ CX, (DI)
	2071	ADDQ $8,DI		2111	ADDQ $8, DI
	2072			2112
n	2073	MOVQ (SI),CX	n	2113	MOVQ (SI), CX
	2074	ADDQ $8,SI		2114	ADDQ $8, SI
	2075	MOVQ CX,(DI)		2115	MOVQ CX, (DI)
	2076	ADDQ $8,DI		2116	ADDQ $8, DI
	2077			2117
n	2078	MOVQ (SI),CX	n	2118	MOVQ (SI), CX
	2079	ADDQ $8,SI		2119	ADDQ $8, SI
	2080	MOVQ CX,(DI)		2120	MOVQ CX, (DI)
	2081	ADDQ $8,DI		2121	ADDQ $8, DI
	2082			2122
n	2083	MOVQ (SI),CX	n	2123	MOVQ (SI), CX
	2084	ADDQ $8,SI		2124	ADDQ $8, SI
	2085	MOVQ CX,(DI)		2125	MOVQ CX, (DI)
	2086	ADDQ $8,DI		2126	ADDQ $8, DI
	2087			2127
n	2088	MOVQ (SI),CX	n	2128	MOVQ (SI), CX
	2089	ADDQ $8,SI		2129	ADDQ $8, SI
	2090	MOVQ CX,(DI)		2130	MOVQ CX, (DI)
	2091	ADDQ $8,DI		2131	ADDQ $8, DI
	2092			2132
n	2093	MOVQ (SI),CX	n	2133	MOVQ (SI), CX
	2094	ADDQ $8,SI		2134	ADDQ $8, SI
	2095	MOVQ CX,(DI)		2135	MOVQ CX, (DI)
	2096	ADDQ $8,DI		2136	ADDQ $8, DI
	2097			2137
n	2098	MOVQ (SI),CX	n	2138	MOVQ (SI), CX
	2099	ADDQ $8,SI		2139	ADDQ $8, SI
	2100	MOVQ CX,(DI)		2140	MOVQ CX, (DI)
	2101	ADDQ $8,DI		2141	ADDQ $8, DI
	2102			2142
n	2103	MOVQ (SI),CX	n	2143	MOVQ (SI), CX
	2104	ADDQ $8,SI		2144	ADDQ $8, SI
	2105	MOVQ CX,(DI)		2145	MOVQ CX, (DI)
	2106	ADDQ $8,DI		2146	ADDQ $8, DI
	2107			2147
n	2108	MOVQ (SI),CX	n	2148	MOVQ (SI), CX
	2109	ADDQ $8,SI		2149	ADDQ $8, SI
	2110	MOVQ CX,(DI)		2150	MOVQ CX, (DI)
	2111	ADDQ $8,DI		2151	ADDQ $8, DI
	2112			2152
n	2113	MOVQ (SI),CX	n	2153	MOVQ (SI), CX
	2114	ADDQ $8,SI		2154	ADDQ $8, SI
	2115	MOVQ CX,(DI)		2155	MOVQ CX, (DI)
	2116	ADDQ $8,DI		2156	ADDQ $8, DI
	2117			2157
n	2118	MOVQ (SI),CX	n	2158	MOVQ (SI), CX
	2119	ADDQ $8,SI		2159	ADDQ $8, SI
	2120	MOVQ CX,(DI)		2160	MOVQ CX, (DI)
	2121	ADDQ $8,DI		2161	ADDQ $8, DI
	2122			2162
n	2123	MOVQ (SI),CX	n	2163	MOVQ (SI), CX
	2124	ADDQ $8,SI		2164	ADDQ $8, SI
	2125	MOVQ CX,(DI)		2165	MOVQ CX, (DI)
	2126	ADDQ $8,DI		2166	ADDQ $8, DI
	2127			2167
n	2128	MOVQ (SI),CX	n	2168	MOVQ (SI), CX
	2129	ADDQ $8,SI		2169	ADDQ $8, SI
	2130	MOVQ CX,(DI)		2170	MOVQ CX, (DI)
	2131	ADDQ $8,DI		2171	ADDQ $8, DI
	2132			2172
n	2133	MOVQ (SI),CX	n	2173	MOVQ (SI), CX
	2134	ADDQ $8,SI		2174	ADDQ $8, SI
	2135	MOVQ CX,(DI)		2175	MOVQ CX, (DI)
	2136	ADDQ $8,DI		2176	ADDQ $8, DI
	2137			2177
n	2138	MOVQ (SI),CX	n	2178	MOVQ (SI), CX
	2139	ADDQ $8,SI		2179	ADDQ $8, SI
	2140	MOVQ CX,(DI)		2180	MOVQ CX, (DI)
	2141	ADDQ $8,DI		2181	ADDQ $8, DI
	2142			2182
n	2143	MOVQ (SI),CX	n	2183	MOVQ (SI), CX
	2144	ADDQ $8,SI		2184	ADDQ $8, SI
	2145	MOVQ CX,(DI)		2185	MOVQ CX, (DI)
	2146	ADDQ $8,DI		2186	ADDQ $8, DI
	2147			2187
n	2148	MOVQ (SI),CX	n	2188	MOVQ (SI), CX
	2149	ADDQ $8,SI		2189	ADDQ $8, SI
	2150	MOVQ CX,(DI)		2190	MOVQ CX, (DI)
	2151	ADDQ $8,DI		2191	ADDQ $8, DI
	2152			2192
n	2153	MOVQ (SI),CX	n	2193	MOVQ (SI), CX
	2154	ADDQ $8,SI		2194	ADDQ $8, SI
	2155	MOVQ CX,(DI)		2195	MOVQ CX, (DI)
	2156	ADDQ $8,DI		2196	ADDQ $8, DI
	2157			2197
n	2158	MOVQ (SI),CX	n	2198	MOVQ (SI), CX
	2159	ADDQ $8,SI		2199	ADDQ $8, SI
	2160	MOVQ CX,(DI)		2200	MOVQ CX, (DI)
	2161	ADDQ $8,DI		2201	ADDQ $8, DI
	2162			2202
n	2163	MOVQ (SI),CX	n	2203	MOVQ (SI), CX
	2164	ADDQ $8,SI		2204	ADDQ $8, SI
	2165	MOVQ CX,(DI)		2205	MOVQ CX, (DI)
	2166	ADDQ $8,DI		2206	ADDQ $8, DI
	2167			2207
n	2168	MOVQ (SI),CX	n	2208	MOVQ (SI), CX
	2169	ADDQ $8,SI		2209	ADDQ $8, SI
	2170	MOVQ CX,(DI)		2210	MOVQ CX, (DI)
	2171	ADDQ $8,DI		2211	ADDQ $8, DI
	2172			2212
n	2173	MOVQ (SI),CX	n	2213	MOVQ (SI), CX
	2174	ADDQ $8,SI		2214	ADDQ $8, SI
	2175	MOVQ CX,(DI)		2215	MOVQ CX, (DI)
	2176	ADDQ $8,DI		2216	ADDQ $8, DI
	2177			2217
n	2178	MOVQ (SI),CX	n	2218	MOVQ (SI), CX
	2179	ADDQ $8,SI		2219	ADDQ $8, SI
	2180	MOVQ CX,(DI)		2220	MOVQ CX, (DI)
	2181	ADDQ $8,DI		2221	ADDQ $8, DI
	2182			2222
n	2183	MOVQ (SI),CX	n	2223	MOVQ (SI), CX
	2184	ADDQ $8,SI		2224	ADDQ $8, SI
	2185	MOVQ CX,(DI)		2225	MOVQ CX, (DI)
	2186	ADDQ $8,DI		2226	ADDQ $8, DI
	2187			2227
n	2188	MOVQ (SI),CX	n	2228	MOVQ (SI), CX
	2189	ADDQ $8,SI		2229	ADDQ $8, SI
	2190	MOVQ CX,(DI)		2230	MOVQ CX, (DI)
	2191	ADDQ $8,DI		2231	ADDQ $8, DI
	2192			2232
n	2193	MOVQ (SI),CX	n	2233	MOVQ (SI), CX
	2194	ADDQ $8,SI		2234	ADDQ $8, SI
	2195	MOVQ CX,(DI)		2235	MOVQ CX, (DI)
	2196	ADDQ $8,DI		2236	ADDQ $8, DI
	2197			2237
n	2198	MOVQ (SI),CX	n	2238	MOVQ (SI), CX
	2199	ADDQ $8,SI		2239	ADDQ $8, SI
	2200	MOVQ CX,(DI)		2240	MOVQ CX, (DI)
	2201	ADDQ $8,DI		2241	ADDQ $8, DI
	2202			2242
	2203	RET		2243	RET
	2204			2244
	2205	TEXT runtime·fastrand1(SB), NOSPLIT, $0-4		2245	TEXT runtime·fastrand1(SB), NOSPLIT, $0-4
	2206	get_tls(CX)		2246	get_tls(CX)
n	2207	MOVQ g(CX), AX	n	2247	MOVQ g(CX), AX
	2208	MOVQ g_m(AX), AX		2248	MOVQ g_m(AX), AX
	2209	MOVL m_fastrand(AX), DX		2249	MOVL m_fastrand(AX), DX
	2210	ADDL DX, DX		2250	ADDL DX, DX
	2211	MOVL DX, BX		2251	MOVL DX, BX
	2212	XORL $0x88888eef, DX		2252	XORL $0x88888eef, DX
	2213	CMOVLMI BX, DX		2253	CMOVLMI BX, DX
	2214	MOVL DX, m_fastrand(AX)		2254	MOVL DX, m_fastrand(AX)
	2215	MOVL DX, ret+0(FP)		2255	MOVL DX, ret+0(FP)
	2216	RET		2256	RET
	2217			2257
	2218	TEXT runtime·return0(SB), NOSPLIT, $0		2258	TEXT runtime·return0(SB), NOSPLIT, $0
n	2219	MOVL $0, AX	n	2259	MOVL $0, AX
	2220	RET		2260	RET
n	2221		n
	2222			2261
	2223	// Called from cgo wrappers, this function returns g->m->curg.stack.hi.		2262	// Called from cgo wrappers, this function returns g->m->curg.stack.hi.
	2224	// Must obey the gcc calling convention.		2263	// Must obey the gcc calling convention.
n	2225	TEXT _cgo_topofstack(SB),NOSPLIT,$0	n	2264	TEXT _cgo_topofstack(SB), NOSPLIT, $0
	2226	get_tls(CX)		2265	get_tls(CX)
n	2227	MOVQ g(CX), AX	n	2266	MOVQ g(CX), AX
	2228	MOVQ g_m(AX), AX		2267	MOVQ g_m(AX), AX
	2229	MOVQ m_curg(AX), AX		2268	MOVQ m_curg(AX), AX
	2230	MOVQ (g_stack+stack_hi)(AX), AX		2269	MOVQ (g_stack+stack_hi)(AX), AX
	2231	RET		2270	RET
	2232			2271
	2233	// The top-most function running on a goroutine		2272	// The top-most function running on a goroutine
	2234	// returns to goexit+PCQuantum.		2273	// returns to goexit+PCQuantum.
t	2235	TEXT runtime·goexit(SB),NOSPLIT,$0-0	t	2274	TEXT runtime·goexit(SB), NOSPLIT, $0-0
	2236	BYTE $0x90 // NOP		2275	BYTE $0x90 // NOP
	2237	CALL runtime·goexit1(SB) // does not return		2276	CALL runtime·goexit1(SB) // does not return