cpu_x86.go

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build 386 || amd64 || amd64p32
     6  
     7  package cpu
     8  
     9  import "runtime"
    10  
    11  const cacheLineSize = 64
    12  
    13  func initOptions() {
    14  	options = []option{
    15  		{Name: "adx", Feature: &X86.HasADX},
    16  		{Name: "aes", Feature: &X86.HasAES},
    17  		{Name: "avx", Feature: &X86.HasAVX},
    18  		{Name: "avx2", Feature: &X86.HasAVX2},
    19  		{Name: "avx512", Feature: &X86.HasAVX512},
    20  		{Name: "avx512f", Feature: &X86.HasAVX512F},
    21  		{Name: "avx512cd", Feature: &X86.HasAVX512CD},
    22  		{Name: "avx512er", Feature: &X86.HasAVX512ER},
    23  		{Name: "avx512pf", Feature: &X86.HasAVX512PF},
    24  		{Name: "avx512vl", Feature: &X86.HasAVX512VL},
    25  		{Name: "avx512bw", Feature: &X86.HasAVX512BW},
    26  		{Name: "avx512dq", Feature: &X86.HasAVX512DQ},
    27  		{Name: "avx512ifma", Feature: &X86.HasAVX512IFMA},
    28  		{Name: "avx512vbmi", Feature: &X86.HasAVX512VBMI},
    29  		{Name: "avx512vnniw", Feature: &X86.HasAVX5124VNNIW},
    30  		{Name: "avx5124fmaps", Feature: &X86.HasAVX5124FMAPS},
    31  		{Name: "avx512vpopcntdq", Feature: &X86.HasAVX512VPOPCNTDQ},
    32  		{Name: "avx512vpclmulqdq", Feature: &X86.HasAVX512VPCLMULQDQ},
    33  		{Name: "avx512vnni", Feature: &X86.HasAVX512VNNI},
    34  		{Name: "avx512gfni", Feature: &X86.HasAVX512GFNI},
    35  		{Name: "avx512vaes", Feature: &X86.HasAVX512VAES},
    36  		{Name: "avx512vbmi2", Feature: &X86.HasAVX512VBMI2},
    37  		{Name: "avx512bitalg", Feature: &X86.HasAVX512BITALG},
    38  		{Name: "avx512bf16", Feature: &X86.HasAVX512BF16},
    39  		{Name: "amxtile", Feature: &X86.HasAMXTile},
    40  		{Name: "amxint8", Feature: &X86.HasAMXInt8},
    41  		{Name: "amxbf16", Feature: &X86.HasAMXBF16},
    42  		{Name: "bmi1", Feature: &X86.HasBMI1},
    43  		{Name: "bmi2", Feature: &X86.HasBMI2},
    44  		{Name: "cx16", Feature: &X86.HasCX16},
    45  		{Name: "erms", Feature: &X86.HasERMS},
    46  		{Name: "fma", Feature: &X86.HasFMA},
    47  		{Name: "osxsave", Feature: &X86.HasOSXSAVE},
    48  		{Name: "pclmulqdq", Feature: &X86.HasPCLMULQDQ},
    49  		{Name: "popcnt", Feature: &X86.HasPOPCNT},
    50  		{Name: "rdrand", Feature: &X86.HasRDRAND},
    51  		{Name: "rdseed", Feature: &X86.HasRDSEED},
    52  		{Name: "sse3", Feature: &X86.HasSSE3},
    53  		{Name: "sse41", Feature: &X86.HasSSE41},
    54  		{Name: "sse42", Feature: &X86.HasSSE42},
    55  		{Name: "ssse3", Feature: &X86.HasSSSE3},
    56  		{Name: "avxifma", Feature: &X86.HasAVXIFMA},
    57  		{Name: "avxvnni", Feature: &X86.HasAVXVNNI},
    58  		{Name: "avxvnniint8", Feature: &X86.HasAVXVNNIInt8},
    59  
    60  		// These capabilities should always be enabled on amd64:
    61  		{Name: "sse2", Feature: &X86.HasSSE2, Required: runtime.GOARCH == "amd64"},
    62  	}
    63  }
    64  
    65  func archInit() {
    66  
    67  	// From internal/cpu
    68  	const (
    69  		// eax bits
    70  		cpuid_AVXVNNI = 1 << 4
    71  
    72  		// ecx bits
    73  		cpuid_SSE3            = 1 << 0
    74  		cpuid_PCLMULQDQ       = 1 << 1
    75  		cpuid_AVX512VBMI      = 1 << 1
    76  		cpuid_AVX512VBMI2     = 1 << 6
    77  		cpuid_SSSE3           = 1 << 9
    78  		cpuid_AVX512GFNI      = 1 << 8
    79  		cpuid_AVX512VAES      = 1 << 9
    80  		cpuid_AVX512VNNI      = 1 << 11
    81  		cpuid_AVX512BITALG    = 1 << 12
    82  		cpuid_FMA             = 1 << 12
    83  		cpuid_AVX512VPOPCNTDQ = 1 << 14
    84  		cpuid_SSE41           = 1 << 19
    85  		cpuid_SSE42           = 1 << 20
    86  		cpuid_POPCNT          = 1 << 23
    87  		cpuid_AES             = 1 << 25
    88  		cpuid_OSXSAVE         = 1 << 27
    89  		cpuid_AVX             = 1 << 28
    90  
    91  		// "Extended Feature Flag" bits returned in EBX for CPUID EAX=0x7 ECX=0x0
    92  		cpuid_BMI1     = 1 << 3
    93  		cpuid_AVX2     = 1 << 5
    94  		cpuid_BMI2     = 1 << 8
    95  		cpuid_ERMS     = 1 << 9
    96  		cpuid_AVX512F  = 1 << 16
    97  		cpuid_AVX512DQ = 1 << 17
    98  		cpuid_ADX      = 1 << 19
    99  		cpuid_AVX512CD = 1 << 28
   100  		cpuid_SHA      = 1 << 29
   101  		cpuid_AVX512BW = 1 << 30
   102  		cpuid_AVX512VL = 1 << 31
   103  
   104  		// "Extended Feature Flag" bits returned in ECX for CPUID EAX=0x7 ECX=0x0
   105  		cpuid_AVX512_VBMI      = 1 << 1
   106  		cpuid_AVX512_VBMI2     = 1 << 6
   107  		cpuid_GFNI             = 1 << 8
   108  		cpuid_AVX512VPCLMULQDQ = 1 << 10
   109  		cpuid_AVX512_BITALG    = 1 << 12
   110  
   111  		// edx bits
   112  		cpuid_FSRM = 1 << 4
   113  		// edx bits for CPUID 0x80000001
   114  		cpuid_RDTSCP = 1 << 27
   115  	)
   116  	// Additional constants not in internal/cpu
   117  	const (
   118  		// eax=1: edx
   119  		cpuid_SSE2 = 1 << 26
   120  		// eax=1: ecx
   121  		cpuid_CX16   = 1 << 13
   122  		cpuid_RDRAND = 1 << 30
   123  		// eax=7,ecx=0: ebx
   124  		cpuid_RDSEED     = 1 << 18
   125  		cpuid_AVX512IFMA = 1 << 21
   126  		cpuid_AVX512PF   = 1 << 26
   127  		cpuid_AVX512ER   = 1 << 27
   128  		// eax=7,ecx=0: edx
   129  		cpuid_AVX5124VNNIW = 1 << 2
   130  		cpuid_AVX5124FMAPS = 1 << 3
   131  		cpuid_AMXBF16      = 1 << 22
   132  		cpuid_AMXTile      = 1 << 24
   133  		cpuid_AMXInt8      = 1 << 25
   134  		// eax=7,ecx=1: eax
   135  		cpuid_AVX512BF16 = 1 << 5
   136  		cpuid_AVXIFMA    = 1 << 23
   137  		// eax=7,ecx=1: edx
   138  		cpuid_AVXVNNIInt8 = 1 << 4
   139  	)
   140  
   141  	Initialized = true
   142  
   143  	maxID, _, _, _ := cpuid(0, 0)
   144  
   145  	if maxID < 1 {
   146  		return
   147  	}
   148  
   149  	_, _, ecx1, edx1 := cpuid(1, 0)
   150  	X86.HasSSE2 = isSet(edx1, cpuid_SSE2)
   151  
   152  	X86.HasSSE3 = isSet(ecx1, cpuid_SSE3)
   153  	X86.HasPCLMULQDQ = isSet(ecx1, cpuid_PCLMULQDQ)
   154  	X86.HasSSSE3 = isSet(ecx1, cpuid_SSSE3)
   155  	X86.HasFMA = isSet(ecx1, cpuid_FMA)
   156  	X86.HasCX16 = isSet(ecx1, cpuid_CX16)
   157  	X86.HasSSE41 = isSet(ecx1, cpuid_SSE41)
   158  	X86.HasSSE42 = isSet(ecx1, cpuid_SSE42)
   159  	X86.HasPOPCNT = isSet(ecx1, cpuid_POPCNT)
   160  	X86.HasAES = isSet(ecx1, cpuid_AES)
   161  	X86.HasOSXSAVE = isSet(ecx1, cpuid_OSXSAVE)
   162  	X86.HasRDRAND = isSet(ecx1, cpuid_RDRAND)
   163  
   164  	var osSupportsAVX, osSupportsAVX512 bool
   165  	// For XGETBV, OSXSAVE bit is required and sufficient.
   166  	if X86.HasOSXSAVE {
   167  		eax, _ := xgetbv()
   168  		// Check if XMM and YMM registers have OS support.
   169  		osSupportsAVX = isSet(eax, 1<<1) && isSet(eax, 1<<2)
   170  
   171  		if runtime.GOOS == "darwin" {
   172  			// Darwin requires special AVX512 checks, see cpu_darwin_x86.go
   173  			osSupportsAVX512 = osSupportsAVX && darwinSupportsAVX512()
   174  		} else {
   175  			// Check if OPMASK and ZMM registers have OS support.
   176  			osSupportsAVX512 = osSupportsAVX && isSet(eax, 1<<5) && isSet(eax, 1<<6) && isSet(eax, 1<<7)
   177  		}
   178  	}
   179  
   180  	X86.HasAVX = isSet(ecx1, cpuid_AVX) && osSupportsAVX
   181  
   182  	if maxID < 7 {
   183  		return
   184  	}
   185  
   186  	eax7, ebx7, ecx7, edx7 := cpuid(7, 0)
   187  	X86.HasBMI1 = isSet(ebx7, cpuid_BMI1)
   188  	X86.HasAVX2 = isSet(ebx7, cpuid_AVX2) && osSupportsAVX
   189  	X86.HasBMI2 = isSet(ebx7, cpuid_BMI2)
   190  	X86.HasERMS = isSet(ebx7, cpuid_ERMS)
   191  	X86.HasRDSEED = isSet(ebx7, cpuid_RDSEED)
   192  	X86.HasADX = isSet(ebx7, cpuid_ADX)
   193  
   194  	X86.HasAVX512 = isSet(ebx7, cpuid_AVX512F) && osSupportsAVX512 // Because avx-512 foundation is the core required extension
   195  	if X86.HasAVX512 {
   196  		X86.HasAVX512F = true
   197  		X86.HasAVX512CD = isSet(ebx7, cpuid_AVX512CD)
   198  		X86.HasAVX512ER = isSet(ebx7, cpuid_AVX512ER)
   199  		X86.HasAVX512PF = isSet(ebx7, cpuid_AVX512PF)
   200  		X86.HasAVX512VL = isSet(ebx7, cpuid_AVX512VL)
   201  		X86.HasAVX512BW = isSet(ebx7, cpuid_AVX512BW)
   202  		X86.HasAVX512DQ = isSet(ebx7, cpuid_AVX512DQ)
   203  		X86.HasAVX512IFMA = isSet(ebx7, cpuid_AVX512IFMA)
   204  		X86.HasAVX512VBMI = isSet(ecx7, cpuid_AVX512_VBMI)
   205  		X86.HasAVX5124VNNIW = isSet(edx7, cpuid_AVX5124VNNIW)
   206  		X86.HasAVX5124FMAPS = isSet(edx7, cpuid_AVX5124FMAPS)
   207  		X86.HasAVX512VPOPCNTDQ = isSet(ecx7, cpuid_AVX512VPOPCNTDQ)
   208  		X86.HasAVX512VPCLMULQDQ = isSet(ecx7, cpuid_AVX512VPCLMULQDQ)
   209  		X86.HasAVX512VNNI = isSet(ecx7, cpuid_AVX512VNNI)
   210  		X86.HasAVX512GFNI = isSet(ecx7, cpuid_AVX512GFNI)
   211  		X86.HasAVX512VAES = isSet(ecx7, cpuid_AVX512VAES)
   212  		X86.HasAVX512VBMI2 = isSet(ecx7, cpuid_AVX512VBMI2)
   213  		X86.HasAVX512BITALG = isSet(ecx7, cpuid_AVX512BITALG)
   214  	}
   215  
   216  	X86.HasAMXTile = isSet(edx7, cpuid_AMXTile)
   217  	X86.HasAMXInt8 = isSet(edx7, cpuid_AMXInt8)
   218  	X86.HasAMXBF16 = isSet(edx7, cpuid_AMXBF16)
   219  
   220  	// These features depend on the second level of extended features.
   221  	if eax7 >= 1 {
   222  		eax71, _, _, edx71 := cpuid(7, 1)
   223  		if X86.HasAVX512 {
   224  			X86.HasAVX512BF16 = isSet(eax71, cpuid_AVX512BF16)
   225  		}
   226  		if X86.HasAVX {
   227  			X86.HasAVXIFMA = isSet(eax71, cpuid_AVXIFMA)
   228  			X86.HasAVXVNNI = isSet(eax71, cpuid_AVXVNNI)
   229  			X86.HasAVXVNNIInt8 = isSet(edx71, cpuid_AVXVNNIInt8)
   230  		}
   231  	}
   232  }
   233  
   234  func isSet(hwc uint32, value uint32) bool {
   235  	return hwc&value != 0
   236  }
   237
View as plain text