x86.rs.html -- source
Description: Source to the Rust file `src/libstd/../stdarch/crates/std_detect/src/detect/os/x86.rs`.
Keywords: rust, rustlang, rust-lang

      1
      2
      3
      4
      5
      6
      7
      8
      9
     10
     11
     12
     13
     14
     15
     16
     17
     18
     19
     20
     21
     22
     23
     24
     25
     26
     27
     28
     29
     30
     31
     32
     33
     34
     35
     36
     37
     38
     39
     40
     41
     42
     43
     44
     45
     46
     47
     48
     49
     50
     51
     52
     53
     54
     55
     56
     57
     58
     59
     60
     61
     62
     63
     64
     65
     66
     67
     68
     69
     70
     71
     72
     73
     74
     75
     76
     77
     78
     79
     80
     81
     82
     83
     84
     85
     86
     87
     88
     89
     90
     91
     92
     93
     94
     95
     96
     97
     98
     99
    100
    101
    102
    103
    104
    105
    106
    107
    108
    109
    110
    111
    112
    113
    114
    115
    116
    117
    118
    119
    120
    121
    122
    123
    124
    125
    126
    127
    128
    129
    130
    131
    132
    133
    134
    135
    136
    137
    138
    139
    140
    141
    142
    143
    144
    145
    146
    147
    148
    149
    150
    151
    152
    153
    154
    155
    156
    157
    158
    159
    160
    161
    162
    163
    164
    165
    166
    167
    168
    169
    170
    171
    172
    173
    174
    175
    176
    177
    178
    179
    180
    181
    182
    183
    184
    185
    186
    187
    188
    189
    190
    191
    192
    193
    194
    195
    196
    197
    198
    199
    200
    201
    202
    203
    204
    205
    206
    207
    208
    209
    210
    211
    212
    213
    214
    215
    216
    217
    218
    219
    220
    221
    222
    223
    224
    225
    226
    227
    228
    229
    230
    231
    232
    233
    234
    235
    236
    237
    238
    239
    240
    241
    242
    243
    244
    245
    246
    247
    248
    249
    250
    251
    252
    253

    //! x86 run-time feature detection is OS independent.

    #[cfg(target_arch = "x86")]
    use crate::arch::x86::*;
    #[cfg(target_arch = "x86_64")]
    use crate::arch::x86_64::*;

    use crate::mem;

    use crate::detect::{bit, cache, Feature};

    /// Run-time feature detection on x86 works by using the CPUID instruction.
    ///
    /// The [CPUID Wikipedia page][wiki_cpuid] contains
    /// all the information about which flags to set to query which values, and in
    /// which registers these are reported.
    ///
    /// The definitive references are:
    /// - [Intel 64 and IA-32 Architectures Software Developer's Manual Volume 2:
    ///   Instruction Set Reference, A-Z][intel64_ref].
    /// - [AMD64 Architecture Programmer's Manual, Volume 3: General-Purpose and
    ///   System Instructions][amd64_ref].
    ///
    /// [wiki_cpuid]: https://en.wikipedia.org/wiki/CPUID
    /// [intel64_ref]: http://www.intel.de/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
    /// [amd64_ref]: http://support.amd.com/TechDocs/24594.pdf
    #[allow(clippy::similar_names)]
    pub(crate) fn detect_features() -> cache::Initializer {
        let mut value = cache::Initializer::default();

        // If the x86 CPU does not support the CPUID instruction then it is too
        // old to support any of the currently-detectable features.
        if !has_cpuid() {
            return value;
        }

        // Calling `__cpuid`/`__cpuid_count` from here on is safe because the CPU
        // has `cpuid` support.

        // 0. EAX = 0: Basic Information:
        // - EAX returns the "Highest Function Parameter", that is, the maximum
        // leaf value for subsequent calls of `cpuinfo` in range [0,
        // 0x8000_0000]. - The vendor ID is stored in 12 u8 ascii chars,
        // returned in EBX, EDX, and   ECX (in that order):
        let (max_basic_leaf, vendor_id) = unsafe {
            let CpuidResult {
                eax: max_basic_leaf,
                ebx,
                ecx,
                edx,
            } = __cpuid(0);
            let vendor_id: [[u8; 4]; 3] = [
                mem::transmute(ebx),
                mem::transmute(edx),
                mem::transmute(ecx),
            ];
            let vendor_id: [u8; 12] = mem::transmute(vendor_id);
            (max_basic_leaf, vendor_id)
        };

        if max_basic_leaf < 1 {
            // Earlier Intel 486, CPUID not implemented
            return value;
        }

        // EAX = 1, ECX = 0: Queries "Processor Info and Feature Bits";
        // Contains information about most x86 features.
        let CpuidResult {
            ecx: proc_info_ecx,
            edx: proc_info_edx,
            ..
        } = unsafe { __cpuid(0x0000_0001_u32) };

        // EAX = 7, ECX = 0: Queries "Extended Features";
        // Contains information about bmi,bmi2, and avx2 support.
        let (extended_features_ebx, extended_features_ecx) = if max_basic_leaf >= 7 {
            let CpuidResult { ebx, ecx, .. } = unsafe { __cpuid(0x0000_0007_u32) };
            (ebx, ecx)
        } else {
            (0, 0) // CPUID does not support "Extended Features"
        };

        // EAX = 0x8000_0000, ECX = 0: Get Highest Extended Function Supported
        // - EAX returns the max leaf value for extended information, that is,
        // `cpuid` calls in range [0x8000_0000; u32::MAX]:
        let CpuidResult {
            eax: extended_max_basic_leaf,
            ..
        } = unsafe { __cpuid(0x8000_0000_u32) };

        // EAX = 0x8000_0001, ECX=0: Queries "Extended Processor Info and Feature
        // Bits"
        let extended_proc_info_ecx = if extended_max_basic_leaf >= 1 {
            let CpuidResult { ecx, .. } = unsafe { __cpuid(0x8000_0001_u32) };
            ecx
        } else {
            0
        };

        {
            // borrows value till the end of this scope:
            let mut enable = |r, rb, f| {
                if bit::test(r as usize, rb) {
                    value.set(f as u32);
                }
            };

            enable(proc_info_ecx, 0, Feature::sse3);
            enable(proc_info_ecx, 1, Feature::pclmulqdq);
            enable(proc_info_ecx, 9, Feature::ssse3);
            enable(proc_info_ecx, 13, Feature::cmpxchg16b);
            enable(proc_info_ecx, 19, Feature::sse4_1);
            enable(proc_info_ecx, 20, Feature::sse4_2);
            enable(proc_info_ecx, 23, Feature::popcnt);
            enable(proc_info_ecx, 25, Feature::aes);
            enable(proc_info_ecx, 29, Feature::f16c);
            enable(proc_info_ecx, 30, Feature::rdrand);
            enable(extended_features_ebx, 18, Feature::rdseed);
            enable(extended_features_ebx, 19, Feature::adx);
            enable(extended_features_ebx, 11, Feature::rtm);
            enable(proc_info_edx, 4, Feature::tsc);
            enable(proc_info_edx, 23, Feature::mmx);
            enable(proc_info_edx, 24, Feature::fxsr);
            enable(proc_info_edx, 25, Feature::sse);
            enable(proc_info_edx, 26, Feature::sse2);
            enable(extended_features_ebx, 29, Feature::sha);

            enable(extended_features_ebx, 3, Feature::bmi1);
            enable(extended_features_ebx, 8, Feature::bmi2);

            // `XSAVE` and `AVX` support:
            let cpu_xsave = bit::test(proc_info_ecx as usize, 26);
            if cpu_xsave {
                // 0. Here the CPU supports `XSAVE`.

                // 1. Detect `OSXSAVE`, that is, whether the OS is AVX enabled and
                // supports saving the state of the AVX/AVX2 vector registers on
                // context-switches, see:
                //
                // - [intel: is avx enabled?][is_avx_enabled],
                // - [mozilla: sse.cpp][mozilla_sse_cpp].
                //
                // [is_avx_enabled]: https://software.intel.com/en-us/blogs/2011/04/14/is-avx-enabled
                // [mozilla_sse_cpp]: https://hg.mozilla.org/mozilla-central/file/64bab5cbb9b6/mozglue/build/SSE.cpp#l190
                let cpu_osxsave = bit::test(proc_info_ecx as usize, 27);

                if cpu_osxsave {
                    // 2. The OS must have signaled the CPU that it supports saving and
                    // restoring the:
                    //
                    // * SSE -> `XCR0.SSE[1]`
                    // * AVX -> `XCR0.AVX[2]`
                    // * AVX-512 -> `XCR0.AVX-512[7:5]`.
                    //
                    // by setting the corresponding bits of `XCR0` to `1`.
                    //
                    // This is safe because the CPU supports `xsave`
                    // and the OS has set `osxsave`.
                    let xcr0 = unsafe { _xgetbv(0) };
                    // Test `XCR0.SSE[1]` and `XCR0.AVX[2]` with the mask `0b110 == 6`:
                    let os_avx_support = xcr0 & 6 == 6;
                    // Test `XCR0.AVX-512[7:5]` with the mask `0b1110_0000 == 224`:
                    let os_avx512_support = xcr0 & 224 == 224;

                    // Only if the OS and the CPU support saving/restoring the AVX
                    // registers we enable `xsave` support:
                    if os_avx_support {
                        // See "13.3 ENABLING THE XSAVE FEATURE SET AND XSAVE-ENABLED
                        // FEATURES" in the "Intel® 64 and IA-32 Architectures Software
                        // Developer’s Manual, Volume 1: Basic Architecture":
                        //
                        // "Software enables the XSAVE feature set by setting
                        // CR4.OSXSAVE[bit 18] to 1 (e.g., with the MOV to CR4
                        // instruction). If this bit is 0, execution of any of XGETBV,
                        // XRSTOR, XRSTORS, XSAVE, XSAVEC, XSAVEOPT, XSAVES, and XSETBV
                        // causes an invalid-opcode exception (#UD)"
                        //
                        enable(proc_info_ecx, 26, Feature::xsave);

                        // For `xsaveopt`, `xsavec`, and `xsaves` we need to query:
                        // Processor Extended State Enumeration Sub-leaf (EAX = 0DH,
                        // ECX = 1):
                        if max_basic_leaf >= 0xd {
                            let CpuidResult {
                                eax: proc_extended_state1_eax,
                                ..
                            } = unsafe { __cpuid_count(0xd_u32, 1) };
                            enable(proc_extended_state1_eax, 0, Feature::xsaveopt);
                            enable(proc_extended_state1_eax, 1, Feature::xsavec);
                            enable(proc_extended_state1_eax, 3, Feature::xsaves);
                        }

                        // FMA (uses 256-bit wide registers):
                        enable(proc_info_ecx, 12, Feature::fma);

                        // And AVX/AVX2:
                        enable(proc_info_ecx, 28, Feature::avx);
                        enable(extended_features_ebx, 5, Feature::avx2);

                        // For AVX-512 the OS also needs to support saving/restoring
                        // the extended state, only then we enable AVX-512 support:
                        if os_avx512_support {
                            enable(extended_features_ebx, 16, Feature::avx512f);
                            enable(extended_features_ebx, 17, Feature::avx512dq);
                            enable(extended_features_ebx, 21, Feature::avx512ifma);
                            enable(extended_features_ebx, 26, Feature::avx512pf);
                            enable(extended_features_ebx, 27, Feature::avx512er);
                            enable(extended_features_ebx, 28, Feature::avx512cd);
                            enable(extended_features_ebx, 30, Feature::avx512bw);
                            enable(extended_features_ebx, 31, Feature::avx512vl);
                            enable(extended_features_ecx, 1, Feature::avx512vbmi);
                            enable(extended_features_ecx, 5, Feature::avx512bf16);
                            enable(extended_features_ecx, 6, Feature::avx512vbmi2);
                            enable(extended_features_ecx, 8, Feature::avx512gfni);
                            enable(extended_features_ecx, 8, Feature::avx512vp2intersect);
                            enable(extended_features_ecx, 9, Feature::avx512vaes);
                            enable(extended_features_ecx, 10, Feature::avx512vpclmulqdq);
                            enable(extended_features_ecx, 11, Feature::avx512vnni);
                            enable(extended_features_ecx, 12, Feature::avx512bitalg);
                            enable(extended_features_ecx, 14, Feature::avx512vpopcntdq);
                        }
                    }
                }
            }

            // This detects ABM on AMD CPUs and LZCNT on Intel CPUs.
            // On intel CPUs with popcnt, lzcnt implements the
            // "missing part" of ABM, so we map both to the same
            // internal feature.
            //
            // The `is_x86_feature_detected!("lzcnt")` macro then
            // internally maps to Feature::abm.
            enable(extended_proc_info_ecx, 5, Feature::lzcnt);

            // As Hygon Dhyana originates from AMD technology and shares most of the architecture with
            // AMD's family 17h, but with different CPU Vendor ID("HygonGenuine")/Family series
            // number(Family 18h).
            //
            // For CPUID feature bits, Hygon Dhyana(family 18h) share the same definition with AMD
            // family 17h.
            //
            // Related AMD CPUID specification is https://www.amd.com/system/files/TechDocs/25481.pdf.
            // Related Hygon kernel patch can be found on
            // http://lkml.kernel.org/r/5ce86123a7b9dad925ac583d88d2f921040e859b.1538583282.git.puwen@hygon.cn
            if vendor_id == *b"AuthenticAMD" || vendor_id == *b"HygonGenuine" {
                // These features are available on AMD arch CPUs:
                enable(extended_proc_info_ecx, 6, Feature::sse4a);
                enable(extended_proc_info_ecx, 21, Feature::tbm);
            }
        }

        value
    }