// Copyright 2009 The Go Authors. All rights reserved.
|
// Use of this source code is governed by a BSD-style
|
// license that can be found in the LICENSE file.
|
|
package runtime
|
|
import (
|
"runtime/internal/atomic"
|
"unsafe"
|
)
|
|
// TODO(brainman): should not need those
|
const (
|
_NSIG = 65
|
)
|
|
//go:cgo_import_dynamic runtime._AddVectoredExceptionHandler AddVectoredExceptionHandler%2 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._CloseHandle CloseHandle%1 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._CreateEventA CreateEventA%4 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._CreateIoCompletionPort CreateIoCompletionPort%4 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._CreateThread CreateThread%6 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._CreateWaitableTimerA CreateWaitableTimerA%3 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._DuplicateHandle DuplicateHandle%7 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._ExitProcess ExitProcess%1 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._FreeEnvironmentStringsW FreeEnvironmentStringsW%1 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._GetConsoleMode GetConsoleMode%2 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._GetEnvironmentStringsW GetEnvironmentStringsW%0 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._GetProcAddress GetProcAddress%2 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._GetProcessAffinityMask GetProcessAffinityMask%3 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._GetQueuedCompletionStatus GetQueuedCompletionStatus%5 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._GetStdHandle GetStdHandle%1 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._GetSystemInfo GetSystemInfo%1 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._GetThreadContext GetThreadContext%2 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._LoadLibraryW LoadLibraryW%1 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._LoadLibraryA LoadLibraryA%1 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._ResumeThread ResumeThread%1 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._SetConsoleCtrlHandler SetConsoleCtrlHandler%2 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._SetErrorMode SetErrorMode%1 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._SetEvent SetEvent%1 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._SetProcessPriorityBoost SetProcessPriorityBoost%2 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._SetThreadPriority SetThreadPriority%2 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._SetUnhandledExceptionFilter SetUnhandledExceptionFilter%1 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._SetWaitableTimer SetWaitableTimer%6 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._SuspendThread SuspendThread%1 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._SwitchToThread SwitchToThread%0 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._TlsAlloc TlsAlloc%0 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._VirtualAlloc VirtualAlloc%4 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._VirtualFree VirtualFree%3 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._VirtualQuery VirtualQuery%3 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._WSAGetOverlappedResult WSAGetOverlappedResult%5 "ws2_32.dll"
|
//go:cgo_import_dynamic runtime._WaitForSingleObject WaitForSingleObject%2 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._WriteConsoleW WriteConsoleW%5 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._WriteFile WriteFile%5 "kernel32.dll"
|
//go:cgo_import_dynamic runtime._timeBeginPeriod timeBeginPeriod%1 "winmm.dll"
|
//go:cgo_import_dynamic runtime._timeEndPeriod timeEndPeriod%1 "winmm.dll"
|
|
type stdFunction unsafe.Pointer
|
|
var (
|
// Following syscalls are available on every Windows PC.
|
// All these variables are set by the Windows executable
|
// loader before the Go program starts.
|
_AddVectoredExceptionHandler,
|
_CloseHandle,
|
_CreateEventA,
|
_CreateIoCompletionPort,
|
_CreateThread,
|
_CreateWaitableTimerA,
|
_DuplicateHandle,
|
_ExitProcess,
|
_FreeEnvironmentStringsW,
|
_GetConsoleMode,
|
_GetEnvironmentStringsW,
|
_GetProcAddress,
|
_GetProcessAffinityMask,
|
_GetQueuedCompletionStatus,
|
_GetStdHandle,
|
_GetSystemInfo,
|
_GetSystemTimeAsFileTime,
|
_GetThreadContext,
|
_LoadLibraryW,
|
_LoadLibraryA,
|
_QueryPerformanceCounter,
|
_QueryPerformanceFrequency,
|
_ResumeThread,
|
_SetConsoleCtrlHandler,
|
_SetErrorMode,
|
_SetEvent,
|
_SetProcessPriorityBoost,
|
_SetThreadPriority,
|
_SetUnhandledExceptionFilter,
|
_SetWaitableTimer,
|
_SuspendThread,
|
_SwitchToThread,
|
_TlsAlloc,
|
_VirtualAlloc,
|
_VirtualFree,
|
_VirtualQuery,
|
_WSAGetOverlappedResult,
|
_WaitForSingleObject,
|
_WriteConsoleW,
|
_WriteFile,
|
_timeBeginPeriod,
|
_timeEndPeriod,
|
_ stdFunction
|
|
// Following syscalls are only available on some Windows PCs.
|
// We will load syscalls, if available, before using them.
|
_AddDllDirectory,
|
_AddVectoredContinueHandler,
|
_GetQueuedCompletionStatusEx,
|
_LoadLibraryExW,
|
_ stdFunction
|
|
// Use RtlGenRandom to generate cryptographically random data.
|
// This approach has been recommended by Microsoft (see issue
|
// 15589 for details).
|
// The RtlGenRandom is not listed in advapi32.dll, instead
|
// RtlGenRandom function can be found by searching for SystemFunction036.
|
// Also some versions of Mingw cannot link to SystemFunction036
|
// when building executable as Cgo. So load SystemFunction036
|
// manually during runtime startup.
|
_RtlGenRandom stdFunction
|
|
// Load ntdll.dll manually during startup, otherwise Mingw
|
// links wrong printf function to cgo executable (see issue
|
// 12030 for details).
|
_NtWaitForSingleObject stdFunction
|
)
|
|
// Function to be called by windows CreateThread
|
// to start new os thread.
|
func tstart_stdcall(newm *m) uint32
|
|
func ctrlhandler(_type uint32) uint32
|
|
type mOS struct {
|
waitsema uintptr // semaphore for parking on locks
|
}
|
|
//go:linkname os_sigpipe os.sigpipe
|
func os_sigpipe() {
|
throw("too many writes on closed pipe")
|
}
|
|
// Stubs so tests can link correctly. These should never be called.
|
func open(name *byte, mode, perm int32) int32 {
|
throw("unimplemented")
|
return -1
|
}
|
func closefd(fd int32) int32 {
|
throw("unimplemented")
|
return -1
|
}
|
func read(fd int32, p unsafe.Pointer, n int32) int32 {
|
throw("unimplemented")
|
return -1
|
}
|
|
type sigset struct{}
|
|
// Call a Windows function with stdcall conventions,
|
// and switch to os stack during the call.
|
func asmstdcall(fn unsafe.Pointer)
|
|
var asmstdcallAddr unsafe.Pointer
|
|
func windowsFindfunc(lib uintptr, name []byte) stdFunction {
|
if name[len(name)-1] != 0 {
|
throw("usage")
|
}
|
f := stdcall2(_GetProcAddress, lib, uintptr(unsafe.Pointer(&name[0])))
|
return stdFunction(unsafe.Pointer(f))
|
}
|
|
func loadOptionalSyscalls() {
|
var kernel32dll = []byte("kernel32.dll\000")
|
k32 := stdcall1(_LoadLibraryA, uintptr(unsafe.Pointer(&kernel32dll[0])))
|
if k32 == 0 {
|
throw("kernel32.dll not found")
|
}
|
_AddDllDirectory = windowsFindfunc(k32, []byte("AddDllDirectory\000"))
|
_AddVectoredContinueHandler = windowsFindfunc(k32, []byte("AddVectoredContinueHandler\000"))
|
_GetQueuedCompletionStatusEx = windowsFindfunc(k32, []byte("GetQueuedCompletionStatusEx\000"))
|
_LoadLibraryExW = windowsFindfunc(k32, []byte("LoadLibraryExW\000"))
|
|
var advapi32dll = []byte("advapi32.dll\000")
|
a32 := stdcall1(_LoadLibraryA, uintptr(unsafe.Pointer(&advapi32dll[0])))
|
if a32 == 0 {
|
throw("advapi32.dll not found")
|
}
|
_RtlGenRandom = windowsFindfunc(a32, []byte("SystemFunction036\000"))
|
|
var ntdll = []byte("ntdll.dll\000")
|
n32 := stdcall1(_LoadLibraryA, uintptr(unsafe.Pointer(&ntdll[0])))
|
if n32 == 0 {
|
throw("ntdll.dll not found")
|
}
|
_NtWaitForSingleObject = windowsFindfunc(n32, []byte("NtWaitForSingleObject\000"))
|
|
if GOARCH == "arm" {
|
_QueryPerformanceCounter = windowsFindfunc(k32, []byte("QueryPerformanceCounter\000"))
|
if _QueryPerformanceCounter == nil {
|
throw("could not find QPC syscalls")
|
}
|
}
|
|
if windowsFindfunc(n32, []byte("wine_get_version\000")) != nil {
|
// running on Wine
|
initWine(k32)
|
}
|
}
|
|
//go:nosplit
|
func getLoadLibrary() uintptr {
|
return uintptr(unsafe.Pointer(_LoadLibraryW))
|
}
|
|
//go:nosplit
|
func getLoadLibraryEx() uintptr {
|
return uintptr(unsafe.Pointer(_LoadLibraryExW))
|
}
|
|
//go:nosplit
|
func getGetProcAddress() uintptr {
|
return uintptr(unsafe.Pointer(_GetProcAddress))
|
}
|
|
func getproccount() int32 {
|
var mask, sysmask uintptr
|
ret := stdcall3(_GetProcessAffinityMask, currentProcess, uintptr(unsafe.Pointer(&mask)), uintptr(unsafe.Pointer(&sysmask)))
|
if ret != 0 {
|
n := 0
|
maskbits := int(unsafe.Sizeof(mask) * 8)
|
for i := 0; i < maskbits; i++ {
|
if mask&(1<<uint(i)) != 0 {
|
n++
|
}
|
}
|
if n != 0 {
|
return int32(n)
|
}
|
}
|
// use GetSystemInfo if GetProcessAffinityMask fails
|
var info systeminfo
|
stdcall1(_GetSystemInfo, uintptr(unsafe.Pointer(&info)))
|
return int32(info.dwnumberofprocessors)
|
}
|
|
func getPageSize() uintptr {
|
var info systeminfo
|
stdcall1(_GetSystemInfo, uintptr(unsafe.Pointer(&info)))
|
return uintptr(info.dwpagesize)
|
}
|
|
const (
|
currentProcess = ^uintptr(0) // -1 = current process
|
currentThread = ^uintptr(1) // -2 = current thread
|
)
|
|
// in sys_windows_386.s and sys_windows_amd64.s:
|
func externalthreadhandler()
|
func getlasterror() uint32
|
func setlasterror(err uint32)
|
|
// When loading DLLs, we prefer to use LoadLibraryEx with
|
// LOAD_LIBRARY_SEARCH_* flags, if available. LoadLibraryEx is not
|
// available on old Windows, though, and the LOAD_LIBRARY_SEARCH_*
|
// flags are not available on some versions of Windows without a
|
// security patch.
|
//
|
// https://msdn.microsoft.com/en-us/library/ms684179(v=vs.85).aspx says:
|
// "Windows 7, Windows Server 2008 R2, Windows Vista, and Windows
|
// Server 2008: The LOAD_LIBRARY_SEARCH_* flags are available on
|
// systems that have KB2533623 installed. To determine whether the
|
// flags are available, use GetProcAddress to get the address of the
|
// AddDllDirectory, RemoveDllDirectory, or SetDefaultDllDirectories
|
// function. If GetProcAddress succeeds, the LOAD_LIBRARY_SEARCH_*
|
// flags can be used with LoadLibraryEx."
|
var useLoadLibraryEx bool
|
|
var timeBeginPeriodRetValue uint32
|
|
// osRelaxMinNS indicates that sysmon shouldn't osRelax if the next
|
// timer is less than 60 ms from now. Since osRelaxing may reduce
|
// timer resolution to 15.6 ms, this keeps timer error under roughly 1
|
// part in 4.
|
const osRelaxMinNS = 60 * 1e6
|
|
// osRelax is called by the scheduler when transitioning to and from
|
// all Ps being idle.
|
//
|
// On Windows, it adjusts the system-wide timer resolution. Go needs a
|
// high resolution timer while running and there's little extra cost
|
// if we're already using the CPU, but if all Ps are idle there's no
|
// need to consume extra power to drive the high-res timer.
|
func osRelax(relax bool) uint32 {
|
if relax {
|
return uint32(stdcall1(_timeEndPeriod, 1))
|
} else {
|
return uint32(stdcall1(_timeBeginPeriod, 1))
|
}
|
}
|
|
func osinit() {
|
asmstdcallAddr = unsafe.Pointer(funcPC(asmstdcall))
|
usleep2Addr = unsafe.Pointer(funcPC(usleep2))
|
switchtothreadAddr = unsafe.Pointer(funcPC(switchtothread))
|
|
setBadSignalMsg()
|
|
loadOptionalSyscalls()
|
|
useLoadLibraryEx = (_LoadLibraryExW != nil && _AddDllDirectory != nil)
|
|
disableWER()
|
|
initExceptionHandler()
|
|
stdcall2(_SetConsoleCtrlHandler, funcPC(ctrlhandler), 1)
|
|
timeBeginPeriodRetValue = osRelax(false)
|
|
ncpu = getproccount()
|
|
physPageSize = getPageSize()
|
|
// Windows dynamic priority boosting assumes that a process has different types
|
// of dedicated threads -- GUI, IO, computational, etc. Go processes use
|
// equivalent threads that all do a mix of GUI, IO, computations, etc.
|
// In such context dynamic priority boosting does nothing but harm, so we turn it off.
|
stdcall2(_SetProcessPriorityBoost, currentProcess, 1)
|
}
|
|
func nanotime() int64
|
|
// useQPCTime controls whether time.now and nanotime use QueryPerformanceCounter.
|
// This is only set to 1 when running under Wine.
|
var useQPCTime uint8
|
|
var qpcStartCounter int64
|
var qpcMultiplier int64
|
|
//go:nosplit
|
func nanotimeQPC() int64 {
|
var counter int64 = 0
|
stdcall1(_QueryPerformanceCounter, uintptr(unsafe.Pointer(&counter)))
|
|
// returns number of nanoseconds
|
return (counter - qpcStartCounter) * qpcMultiplier
|
}
|
|
//go:nosplit
|
func nowQPC() (sec int64, nsec int32, mono int64) {
|
var ft int64
|
stdcall1(_GetSystemTimeAsFileTime, uintptr(unsafe.Pointer(&ft)))
|
|
t := (ft - 116444736000000000) * 100
|
|
sec = t / 1000000000
|
nsec = int32(t - sec*1000000000)
|
|
mono = nanotimeQPC()
|
return
|
}
|
|
func initWine(k32 uintptr) {
|
_GetSystemTimeAsFileTime = windowsFindfunc(k32, []byte("GetSystemTimeAsFileTime\000"))
|
if _GetSystemTimeAsFileTime == nil {
|
throw("could not find GetSystemTimeAsFileTime() syscall")
|
}
|
|
_QueryPerformanceCounter = windowsFindfunc(k32, []byte("QueryPerformanceCounter\000"))
|
_QueryPerformanceFrequency = windowsFindfunc(k32, []byte("QueryPerformanceFrequency\000"))
|
if _QueryPerformanceCounter == nil || _QueryPerformanceFrequency == nil {
|
throw("could not find QPC syscalls")
|
}
|
|
// We can not simply fallback to GetSystemTimeAsFileTime() syscall, since its time is not monotonic,
|
// instead we use QueryPerformanceCounter family of syscalls to implement monotonic timer
|
// https://msdn.microsoft.com/en-us/library/windows/desktop/dn553408(v=vs.85).aspx
|
|
var tmp int64
|
stdcall1(_QueryPerformanceFrequency, uintptr(unsafe.Pointer(&tmp)))
|
if tmp == 0 {
|
throw("QueryPerformanceFrequency syscall returned zero, running on unsupported hardware")
|
}
|
|
// This should not overflow, it is a number of ticks of the performance counter per second,
|
// its resolution is at most 10 per usecond (on Wine, even smaller on real hardware), so it will be at most 10 millions here,
|
// panic if overflows.
|
if tmp > (1<<31 - 1) {
|
throw("QueryPerformanceFrequency overflow 32 bit divider, check nosplit discussion to proceed")
|
}
|
qpcFrequency := int32(tmp)
|
stdcall1(_QueryPerformanceCounter, uintptr(unsafe.Pointer(&qpcStartCounter)))
|
|
// Since we are supposed to run this time calls only on Wine, it does not lose precision,
|
// since Wine's timer is kind of emulated at 10 Mhz, so it will be a nice round multiplier of 100
|
// but for general purpose system (like 3.3 Mhz timer on i7) it will not be very precise.
|
// We have to do it this way (or similar), since multiplying QPC counter by 100 millions overflows
|
// int64 and resulted time will always be invalid.
|
qpcMultiplier = int64(timediv(1000000000, qpcFrequency, nil))
|
|
useQPCTime = 1
|
}
|
|
//go:nosplit
|
func getRandomData(r []byte) {
|
n := 0
|
if stdcall2(_RtlGenRandom, uintptr(unsafe.Pointer(&r[0])), uintptr(len(r)))&0xff != 0 {
|
n = len(r)
|
}
|
extendRandom(r, n)
|
}
|
|
func goenvs() {
|
// strings is a pointer to environment variable pairs in the form:
|
// "envA=valA\x00envB=valB\x00\x00" (in UTF-16)
|
// Two consecutive zero bytes end the list.
|
strings := unsafe.Pointer(stdcall0(_GetEnvironmentStringsW))
|
p := (*[1 << 24]uint16)(strings)[:]
|
|
n := 0
|
for from, i := 0, 0; true; i++ {
|
if p[i] == 0 {
|
// empty string marks the end
|
if i == from {
|
break
|
}
|
from = i + 1
|
n++
|
}
|
}
|
envs = make([]string, n)
|
|
for i := range envs {
|
envs[i] = gostringw(&p[0])
|
for p[0] != 0 {
|
p = p[1:]
|
}
|
p = p[1:] // skip nil byte
|
}
|
|
stdcall1(_FreeEnvironmentStringsW, uintptr(strings))
|
}
|
|
// exiting is set to non-zero when the process is exiting.
|
var exiting uint32
|
|
//go:nosplit
|
func exit(code int32) {
|
atomic.Store(&exiting, 1)
|
stdcall1(_ExitProcess, uintptr(code))
|
}
|
|
//go:nosplit
|
func write(fd uintptr, buf unsafe.Pointer, n int32) int32 {
|
const (
|
_STD_OUTPUT_HANDLE = ^uintptr(10) // -11
|
_STD_ERROR_HANDLE = ^uintptr(11) // -12
|
)
|
var handle uintptr
|
switch fd {
|
case 1:
|
handle = stdcall1(_GetStdHandle, _STD_OUTPUT_HANDLE)
|
case 2:
|
handle = stdcall1(_GetStdHandle, _STD_ERROR_HANDLE)
|
default:
|
// assume fd is real windows handle.
|
handle = fd
|
}
|
isASCII := true
|
b := (*[1 << 30]byte)(buf)[:n]
|
for _, x := range b {
|
if x >= 0x80 {
|
isASCII = false
|
break
|
}
|
}
|
|
if !isASCII {
|
var m uint32
|
isConsole := stdcall2(_GetConsoleMode, handle, uintptr(unsafe.Pointer(&m))) != 0
|
// If this is a console output, various non-unicode code pages can be in use.
|
// Use the dedicated WriteConsole call to ensure unicode is printed correctly.
|
if isConsole {
|
return int32(writeConsole(handle, buf, n))
|
}
|
}
|
var written uint32
|
stdcall5(_WriteFile, handle, uintptr(buf), uintptr(n), uintptr(unsafe.Pointer(&written)), 0)
|
return int32(written)
|
}
|
|
var (
|
utf16ConsoleBack [1000]uint16
|
utf16ConsoleBackLock mutex
|
)
|
|
// writeConsole writes bufLen bytes from buf to the console File.
|
// It returns the number of bytes written.
|
func writeConsole(handle uintptr, buf unsafe.Pointer, bufLen int32) int {
|
const surr2 = (surrogateMin + surrogateMax + 1) / 2
|
|
// Do not use defer for unlock. May cause issues when printing a panic.
|
lock(&utf16ConsoleBackLock)
|
|
b := (*[1 << 30]byte)(buf)[:bufLen]
|
s := *(*string)(unsafe.Pointer(&b))
|
|
utf16tmp := utf16ConsoleBack[:]
|
|
total := len(s)
|
w := 0
|
for _, r := range s {
|
if w >= len(utf16tmp)-2 {
|
writeConsoleUTF16(handle, utf16tmp[:w])
|
w = 0
|
}
|
if r < 0x10000 {
|
utf16tmp[w] = uint16(r)
|
w++
|
} else {
|
r -= 0x10000
|
utf16tmp[w] = surrogateMin + uint16(r>>10)&0x3ff
|
utf16tmp[w+1] = surr2 + uint16(r)&0x3ff
|
w += 2
|
}
|
}
|
writeConsoleUTF16(handle, utf16tmp[:w])
|
unlock(&utf16ConsoleBackLock)
|
return total
|
}
|
|
// writeConsoleUTF16 is the dedicated windows calls that correctly prints
|
// to the console regardless of the current code page. Input is utf-16 code points.
|
// The handle must be a console handle.
|
func writeConsoleUTF16(handle uintptr, b []uint16) {
|
l := uint32(len(b))
|
if l == 0 {
|
return
|
}
|
var written uint32
|
stdcall5(_WriteConsoleW,
|
handle,
|
uintptr(unsafe.Pointer(&b[0])),
|
uintptr(l),
|
uintptr(unsafe.Pointer(&written)),
|
0,
|
)
|
return
|
}
|
|
//go:nosplit
|
func semasleep(ns int64) int32 {
|
const (
|
_WAIT_ABANDONED = 0x00000080
|
_WAIT_OBJECT_0 = 0x00000000
|
_WAIT_TIMEOUT = 0x00000102
|
_WAIT_FAILED = 0xFFFFFFFF
|
)
|
|
// store ms in ns to save stack space
|
if ns < 0 {
|
ns = _INFINITE
|
} else {
|
ns = int64(timediv(ns, 1000000, nil))
|
if ns == 0 {
|
ns = 1
|
}
|
}
|
|
result := stdcall2(_WaitForSingleObject, getg().m.waitsema, uintptr(ns))
|
switch result {
|
case _WAIT_OBJECT_0: //signaled
|
return 0
|
|
case _WAIT_TIMEOUT:
|
return -1
|
|
case _WAIT_ABANDONED:
|
systemstack(func() {
|
throw("runtime.semasleep wait_abandoned")
|
})
|
|
case _WAIT_FAILED:
|
systemstack(func() {
|
print("runtime: waitforsingleobject wait_failed; errno=", getlasterror(), "\n")
|
throw("runtime.semasleep wait_failed")
|
})
|
|
default:
|
systemstack(func() {
|
print("runtime: waitforsingleobject unexpected; result=", result, "\n")
|
throw("runtime.semasleep unexpected")
|
})
|
}
|
|
return -1 // unreachable
|
}
|
|
//go:nosplit
|
func semawakeup(mp *m) {
|
if stdcall1(_SetEvent, mp.waitsema) == 0 {
|
systemstack(func() {
|
print("runtime: setevent failed; errno=", getlasterror(), "\n")
|
throw("runtime.semawakeup")
|
})
|
}
|
}
|
|
//go:nosplit
|
func semacreate(mp *m) {
|
if mp.waitsema != 0 {
|
return
|
}
|
mp.waitsema = stdcall4(_CreateEventA, 0, 0, 0, 0)
|
if mp.waitsema == 0 {
|
systemstack(func() {
|
print("runtime: createevent failed; errno=", getlasterror(), "\n")
|
throw("runtime.semacreate")
|
})
|
}
|
}
|
|
// May run with m.p==nil, so write barriers are not allowed. This
|
// function is called by newosproc0, so it is also required to
|
// operate without stack guards.
|
//go:nowritebarrierrec
|
//go:nosplit
|
func newosproc(mp *m) {
|
// We pass 0 for the stack size to use the default for this binary.
|
thandle := stdcall6(_CreateThread, 0, 0,
|
funcPC(tstart_stdcall), uintptr(unsafe.Pointer(mp)),
|
0, 0)
|
|
if thandle == 0 {
|
if atomic.Load(&exiting) != 0 {
|
// CreateThread may fail if called
|
// concurrently with ExitProcess. If this
|
// happens, just freeze this thread and let
|
// the process exit. See issue #18253.
|
lock(&deadlock)
|
lock(&deadlock)
|
}
|
print("runtime: failed to create new OS thread (have ", mcount(), " already; errno=", getlasterror(), ")\n")
|
throw("runtime.newosproc")
|
}
|
|
// Close thandle to avoid leaking the thread object if it exits.
|
stdcall1(_CloseHandle, thandle)
|
}
|
|
// Used by the C library build mode. On Linux this function would allocate a
|
// stack, but that's not necessary for Windows. No stack guards are present
|
// and the GC has not been initialized, so write barriers will fail.
|
//go:nowritebarrierrec
|
//go:nosplit
|
func newosproc0(mp *m, stk unsafe.Pointer) {
|
// TODO: this is completely broken. The args passed to newosproc0 (in asm_amd64.s)
|
// are stacksize and function, not *m and stack.
|
// Check os_linux.go for an implemention that might actually work.
|
throw("bad newosproc0")
|
}
|
|
func exitThread(wait *uint32) {
|
// We should never reach exitThread on Windows because we let
|
// the OS clean up threads.
|
throw("exitThread")
|
}
|
|
// Called to initialize a new m (including the bootstrap m).
|
// Called on the parent thread (main thread in case of bootstrap), can allocate memory.
|
func mpreinit(mp *m) {
|
}
|
|
//go:nosplit
|
func msigsave(mp *m) {
|
}
|
|
//go:nosplit
|
func msigrestore(sigmask sigset) {
|
}
|
|
//go:nosplit
|
//go:nowritebarrierrec
|
func clearSignalHandlers() {
|
}
|
|
//go:nosplit
|
func sigblock() {
|
}
|
|
// Called to initialize a new m (including the bootstrap m).
|
// Called on the new thread, cannot allocate memory.
|
func minit() {
|
var thandle uintptr
|
stdcall7(_DuplicateHandle, currentProcess, currentThread, currentProcess, uintptr(unsafe.Pointer(&thandle)), 0, 0, _DUPLICATE_SAME_ACCESS)
|
atomic.Storeuintptr(&getg().m.thread, thandle)
|
|
// Query the true stack base from the OS. Currently we're
|
// running on a small assumed stack.
|
var mbi memoryBasicInformation
|
res := stdcall3(_VirtualQuery, uintptr(unsafe.Pointer(&mbi)), uintptr(unsafe.Pointer(&mbi)), unsafe.Sizeof(mbi))
|
if res == 0 {
|
print("runtime: VirtualQuery failed; errno=", getlasterror(), "\n")
|
throw("VirtualQuery for stack base failed")
|
}
|
// The system leaves an 8K PAGE_GUARD region at the bottom of
|
// the stack (in theory VirtualQuery isn't supposed to include
|
// that, but it does). Add an additional 8K of slop for
|
// calling C functions that don't have stack checks and for
|
// lastcontinuehandler. We shouldn't be anywhere near this
|
// bound anyway.
|
base := mbi.allocationBase + 16<<10
|
// Sanity check the stack bounds.
|
g0 := getg()
|
if base > g0.stack.hi || g0.stack.hi-base > 64<<20 {
|
print("runtime: g0 stack [", hex(base), ",", hex(g0.stack.hi), ")\n")
|
throw("bad g0 stack")
|
}
|
g0.stack.lo = base
|
g0.stackguard0 = g0.stack.lo + _StackGuard
|
g0.stackguard1 = g0.stackguard0
|
// Sanity check the SP.
|
stackcheck()
|
}
|
|
// Called from dropm to undo the effect of an minit.
|
//go:nosplit
|
func unminit() {
|
tp := &getg().m.thread
|
stdcall1(_CloseHandle, *tp)
|
*tp = 0
|
}
|
|
// Calling stdcall on os stack.
|
// May run during STW, so write barriers are not allowed.
|
//go:nowritebarrier
|
//go:nosplit
|
func stdcall(fn stdFunction) uintptr {
|
gp := getg()
|
mp := gp.m
|
mp.libcall.fn = uintptr(unsafe.Pointer(fn))
|
resetLibcall := false
|
if mp.profilehz != 0 && mp.libcallsp == 0 {
|
// leave pc/sp for cpu profiler
|
mp.libcallg.set(gp)
|
mp.libcallpc = getcallerpc()
|
// sp must be the last, because once async cpu profiler finds
|
// all three values to be non-zero, it will use them
|
mp.libcallsp = getcallersp()
|
resetLibcall = true // See comment in sys_darwin.go:libcCall
|
}
|
asmcgocall(asmstdcallAddr, unsafe.Pointer(&mp.libcall))
|
if resetLibcall {
|
mp.libcallsp = 0
|
}
|
return mp.libcall.r1
|
}
|
|
//go:nosplit
|
func stdcall0(fn stdFunction) uintptr {
|
mp := getg().m
|
mp.libcall.n = 0
|
mp.libcall.args = uintptr(noescape(unsafe.Pointer(&fn))) // it's unused but must be non-nil, otherwise crashes
|
return stdcall(fn)
|
}
|
|
//go:nosplit
|
func stdcall1(fn stdFunction, a0 uintptr) uintptr {
|
mp := getg().m
|
mp.libcall.n = 1
|
mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0)))
|
return stdcall(fn)
|
}
|
|
//go:nosplit
|
func stdcall2(fn stdFunction, a0, a1 uintptr) uintptr {
|
mp := getg().m
|
mp.libcall.n = 2
|
mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0)))
|
return stdcall(fn)
|
}
|
|
//go:nosplit
|
func stdcall3(fn stdFunction, a0, a1, a2 uintptr) uintptr {
|
mp := getg().m
|
mp.libcall.n = 3
|
mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0)))
|
return stdcall(fn)
|
}
|
|
//go:nosplit
|
func stdcall4(fn stdFunction, a0, a1, a2, a3 uintptr) uintptr {
|
mp := getg().m
|
mp.libcall.n = 4
|
mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0)))
|
return stdcall(fn)
|
}
|
|
//go:nosplit
|
func stdcall5(fn stdFunction, a0, a1, a2, a3, a4 uintptr) uintptr {
|
mp := getg().m
|
mp.libcall.n = 5
|
mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0)))
|
return stdcall(fn)
|
}
|
|
//go:nosplit
|
func stdcall6(fn stdFunction, a0, a1, a2, a3, a4, a5 uintptr) uintptr {
|
mp := getg().m
|
mp.libcall.n = 6
|
mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0)))
|
return stdcall(fn)
|
}
|
|
//go:nosplit
|
func stdcall7(fn stdFunction, a0, a1, a2, a3, a4, a5, a6 uintptr) uintptr {
|
mp := getg().m
|
mp.libcall.n = 7
|
mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0)))
|
return stdcall(fn)
|
}
|
|
// in sys_windows_386.s and sys_windows_amd64.s
|
func onosstack(fn unsafe.Pointer, arg uint32)
|
func usleep2(usec uint32)
|
func switchtothread()
|
|
var usleep2Addr unsafe.Pointer
|
var switchtothreadAddr unsafe.Pointer
|
|
//go:nosplit
|
func osyield() {
|
onosstack(switchtothreadAddr, 0)
|
}
|
|
//go:nosplit
|
func usleep(us uint32) {
|
// Have 1us units; want 100ns units.
|
onosstack(usleep2Addr, 10*us)
|
}
|
|
func ctrlhandler1(_type uint32) uint32 {
|
var s uint32
|
|
switch _type {
|
case _CTRL_C_EVENT, _CTRL_BREAK_EVENT:
|
s = _SIGINT
|
default:
|
return 0
|
}
|
|
if sigsend(s) {
|
return 1
|
}
|
exit(2) // SIGINT, SIGTERM, etc
|
return 0
|
}
|
|
// in sys_windows_386.s and sys_windows_amd64.s
|
func profileloop()
|
|
var profiletimer uintptr
|
|
func profilem(mp *m, thread uintptr) {
|
var r *context
|
rbuf := make([]byte, unsafe.Sizeof(*r)+15)
|
|
// align Context to 16 bytes
|
r = (*context)(unsafe.Pointer((uintptr(unsafe.Pointer(&rbuf[15]))) &^ 15))
|
r.contextflags = _CONTEXT_CONTROL
|
stdcall2(_GetThreadContext, thread, uintptr(unsafe.Pointer(r)))
|
|
var gp *g
|
switch GOARCH {
|
default:
|
panic("unsupported architecture")
|
case "arm":
|
tls := &mp.tls[0]
|
gp = **((***g)(unsafe.Pointer(tls)))
|
case "386", "amd64":
|
tls := &mp.tls[0]
|
gp = *((**g)(unsafe.Pointer(tls)))
|
}
|
|
sigprof(r.ip(), r.sp(), r.lr(), gp, mp)
|
}
|
|
func profileloop1(param uintptr) uint32 {
|
stdcall2(_SetThreadPriority, currentThread, _THREAD_PRIORITY_HIGHEST)
|
|
for {
|
stdcall2(_WaitForSingleObject, profiletimer, _INFINITE)
|
first := (*m)(atomic.Loadp(unsafe.Pointer(&allm)))
|
for mp := first; mp != nil; mp = mp.alllink {
|
thread := atomic.Loaduintptr(&mp.thread)
|
// Do not profile threads blocked on Notes,
|
// this includes idle worker threads,
|
// idle timer thread, idle heap scavenger, etc.
|
if thread == 0 || mp.profilehz == 0 || mp.blocked {
|
continue
|
}
|
// mp may exit between the load above and the
|
// SuspendThread, so be careful.
|
if int32(stdcall1(_SuspendThread, thread)) == -1 {
|
// The thread no longer exists.
|
continue
|
}
|
if mp.profilehz != 0 && !mp.blocked {
|
// Pass the thread handle in case mp
|
// was in the process of shutting down.
|
profilem(mp, thread)
|
}
|
stdcall1(_ResumeThread, thread)
|
}
|
}
|
}
|
|
func setProcessCPUProfiler(hz int32) {
|
if profiletimer == 0 {
|
timer := stdcall3(_CreateWaitableTimerA, 0, 0, 0)
|
atomic.Storeuintptr(&profiletimer, timer)
|
thread := stdcall6(_CreateThread, 0, 0, funcPC(profileloop), 0, 0, 0)
|
stdcall2(_SetThreadPriority, thread, _THREAD_PRIORITY_HIGHEST)
|
stdcall1(_CloseHandle, thread)
|
}
|
}
|
|
func setThreadCPUProfiler(hz int32) {
|
ms := int32(0)
|
due := ^int64(^uint64(1 << 63))
|
if hz > 0 {
|
ms = 1000 / hz
|
if ms == 0 {
|
ms = 1
|
}
|
due = int64(ms) * -10000
|
}
|
stdcall6(_SetWaitableTimer, profiletimer, uintptr(unsafe.Pointer(&due)), uintptr(ms), 0, 0, 0)
|
atomic.Store((*uint32)(unsafe.Pointer(&getg().m.profilehz)), uint32(hz))
|
}
|
