      1
      2
      3
      4
      5
      6
      7
      8
      9
     10
     11
     12
     13
     14
     15
     16
     17
     18
     19
     20
     21
     22
     23
     24
     25
     26
     27
     28
     29
     30
     31
     32
     33
     34
     35
     36
     37
     38
     39
     40
     41
     42
     43
     44
     45
     46
     47
     48
     49
     50
     51
     52
     53
     54
     55
     56
     57
     58
     59
     60
     61
     62
     63
     64
     65
     66
     67
     68
     69
     70
     71
     72
     73
     74
     75
     76
     77
     78
     79
     80
     81
     82
     83
     84
     85
     86
     87
     88
     89
     90
     91
     92
     93
     94
     95
     96
     97
     98
     99
    100
    101
    102
    103
    104
    105
    106
    107
    108
    109
    110
    111
    112
    113
    114
    115
    116
    117
    118
    119
    120
    121
    122
    123
    124
    125
    126
    127
    128
    129
    130
    131
    132
    133
    134
    135
    136
    137
    138
    139
    140
    141
    142
    143
    144
    145
    146
    147
    148
    149
    150
    151
    152
    153
    154
    155
    156
    157
    158
    159
    160
    161
    162
    163
    164
    165
    166
    167
    168
    169
    170
    171
    172
    173
    174
    175
    176
    177
    178
    179
    180
    181
    182
    183
    184
    185
    186
    187
    188
    189
    190
    191
    192
    193
    194
    195
    196
    197
    198
    199
    200
    201
    202
    203
    204
    205
    206
    207
    208
    209
    210
    211
    212
    213
    214
    215
    216
    217
    218
    219
    220
    221
    222
    223
    224
    225
    226
    227
    228
    229
    230
    231
    232
    233
    234
    235
    236
    237
    238
    239
    240
    241
    242
    243
    244
    245
    246
    247
    248
    249
    250
    251
    252
    253
    254
    255
    256
    257
    258
    259
    260
    261
    262
    263
    264
    265
    266
    267
    268
    269
    270
    271
    272
    273
    274
    275
    276
    277
    278
    279
    280
    281
    282
    283
    284
    285
    286
    287
    288
    289
    290
    291
    292
    293
    294
    295
    296
    297
    298
    299
    300
    301
    302
    303
    304
    305
    306
    307
    308
    309
    310
    311
    312
    313
    314
    315
    316
    317
    318
    319
    320
    321
    322
    323
    324
    325
    326
    327
    328
    329
    330
    331
    332
    333
    334
    335
    336
    337
    338
    339
    340
    341
    342
    343
    344
    345
    346
    347
    348

    //! x86 run-time feature detection is OS independent.

    use core::prelude::v1::*;
    use core::mem;
    #[cfg(target_arch = "x86")]
    use coresimd::arch::x86::*;
    #[cfg(target_arch = "x86_64")]
    use coresimd::arch::x86_64::*;

    use arch::detect::Feature;
    use arch::detect::cache;
    use arch::detect::bit;

    /// Performs run-time feature detection.
    #[inline]
    pub fn check_for(x: Feature) -> bool {
        cache::test(x as u32, detect_features)
    }

    /// Run-time feature detection on x86 works by using the CPUID instruction.
    ///
    /// The [CPUID Wikipedia page][wiki_cpuid] contains
    /// all the information about which flags to set to query which values, and in
    /// which registers these are reported.
    ///
    /// The definitive references are:
    /// - [Intel 64 and IA-32 Architectures Software Developer's Manual Volume 2:
    ///   Instruction Set Reference, A-Z][intel64_ref].
    /// - [AMD64 Architecture Programmer's Manual, Volume 3: General-Purpose and
    ///   System Instructions][amd64_ref].
    ///
    /// [wiki_cpuid]: https://en.wikipedia.org/wiki/CPUID
    /// [intel64_ref]: http://www.intel.de/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
    /// [amd64_ref]: http://support.amd.com/TechDocs/24594.pdf
    #[cfg_attr(feature = "cargo-clippy", allow(similar_names))]
    fn detect_features() -> cache::Initializer {
        let mut value = cache::Initializer::default();

        // If the x86 CPU does not support the CPUID instruction then it is too
        // old to support any of the currently-detectable features.
        if !has_cpuid() {
            return value;
        }

        // Calling `__cpuid`/`__cpuid_count` from here on is safe because the CPU
        // has `cpuid` support.

        // 0. EAX = 0: Basic Information:
        // - EAX returns the "Highest Function Parameter", that is, the maximum
        // leaf value for subsequent calls of `cpuinfo` in range [0,
        // 0x8000_0000]. - The vendor ID is stored in 12 u8 ascii chars,
        // returned in EBX, EDX, and   ECX (in that order):
        let (max_basic_leaf, vendor_id) = unsafe {
            let CpuidResult {
                eax: max_basic_leaf,
                ebx,
                ecx,
                edx,
            } = __cpuid(0);
            let vendor_id: [[u8; 4]; 3] = [
                mem::transmute(ebx),
                mem::transmute(edx),
                mem::transmute(ecx),
            ];
            let vendor_id: [u8; 12] = mem::transmute(vendor_id);
            (max_basic_leaf, vendor_id)
        };

        if max_basic_leaf < 1 {
            // Earlier Intel 486, CPUID not implemented
            return value;
        }

        // EAX = 1, ECX = 0: Queries "Processor Info and Feature Bits";
        // Contains information about most x86 features.
        let CpuidResult {
            ecx: proc_info_ecx,
            edx: proc_info_edx,
            ..
        } = unsafe { __cpuid(0x0000_0001_u32) };

        // EAX = 7, ECX = 0: Queries "Extended Features";
        // Contains information about bmi,bmi2, and avx2 support.
        let (extended_features_ebx, extended_features_ecx) = if max_basic_leaf >= 7
        {
            let CpuidResult { ebx, ecx, .. } = unsafe { __cpuid(0x0000_0007_u32) };
            (ebx, ecx)
        } else {
            (0, 0) // CPUID does not support "Extended Features"
        };

        // EAX = 0x8000_0000, ECX = 0: Get Highest Extended Function Supported
        // - EAX returns the max leaf value for extended information, that is,
        // `cpuid` calls in range [0x8000_0000; u32::MAX]:
        let CpuidResult {
            eax: extended_max_basic_leaf,
            ..
        } = unsafe { __cpuid(0x8000_0000_u32) };

        // EAX = 0x8000_0001, ECX=0: Queries "Extended Processor Info and Feature
        // Bits"
        let extended_proc_info_ecx = if extended_max_basic_leaf >= 1 {
            let CpuidResult { ecx, .. } = unsafe { __cpuid(0x8000_0001_u32) };
            ecx
        } else {
            0
        };

        {
            // borrows value till the end of this scope:
            let mut enable = |r, rb, f| {
                if bit::test(r as usize, rb) {
                    value.set(f as u32);
                }
            };

            enable(proc_info_ecx, 0, Feature::sse3);
            enable(proc_info_ecx, 9, Feature::ssse3);
            enable(proc_info_ecx, 19, Feature::sse4_1);
            enable(proc_info_ecx, 20, Feature::sse4_2);
            enable(proc_info_ecx, 23, Feature::popcnt);
            enable(proc_info_ecx, 25, Feature::aes);
            enable(proc_info_ecx, 1, Feature::pclmulqdq);
            enable(proc_info_ecx, 30, Feature::rdrand);
            enable(extended_features_ebx, 18, Feature::rdseed);
            enable(proc_info_edx, 4, Feature::tsc);
            enable(proc_info_edx, 23, Feature::mmx);
            enable(proc_info_edx, 24, Feature::fxsr);
            enable(proc_info_edx, 25, Feature::sse);
            enable(proc_info_edx, 26, Feature::sse2);
            enable(extended_features_ebx, 29, Feature::sha);

            enable(extended_features_ebx, 3, Feature::bmi);
            enable(extended_features_ebx, 8, Feature::bmi2);

            // `XSAVE` and `AVX` support:
            let cpu_xsave = bit::test(proc_info_ecx as usize, 26);
            if cpu_xsave {
                // 0. Here the CPU supports `XSAVE`.

                // 1. Detect `OSXSAVE`, that is, whether the OS is AVX enabled and
                // supports saving the state of the AVX/AVX2 vector registers on
                // context-switches, see:
                //
                // - [intel: is avx enabled?][is_avx_enabled],
                // - [mozilla: sse.cpp][mozilla_sse_cpp].
                //
                // [is_avx_enabled]: https://software.intel.com/en-us/blogs/2011/04/14/is-avx-enabled
                // [mozilla_sse_cpp]: https://hg.mozilla.org/mozilla-central/file/64bab5cbb9b6/mozglue/build/SSE.cpp#l190
                let cpu_osxsave = bit::test(proc_info_ecx as usize, 27);

                if cpu_osxsave {
                    // 2. The OS must have signaled the CPU that it supports saving and
                    // restoring the:
                    //
                    // * SSE -> `XCR0.SSE[1]`
                    // * AVX -> `XCR0.AVX[2]`
                    // * AVX-512 -> `XCR0.AVX-512[7:5]`.
                    //
                    // by setting the corresponding bits of `XCR0` to `1`.
                    //
                    // This is safe because the CPU supports `xsave`
                    // and the OS has set `osxsave`.
                    let xcr0 = unsafe { _xgetbv(0) };
                    // Test `XCR0.SSE[1]` and `XCR0.AVX[2]` with the mask `0b110 == 6`:
                    let os_avx_support = xcr0 & 6 == 6;
                    // Test `XCR0.AVX-512[7:5]` with the mask `0b1110_0000 == 224`:
                    let os_avx512_support = xcr0 & 224 == 224;

                    // Only if the OS and the CPU support saving/restoring the AVX
                    // registers we enable `xsave` support:
                    if os_avx_support {
                        // See "13.3 ENABLING THE XSAVE FEATURE SET AND XSAVE-ENABLED
                        // FEATURES" in the "Intel® 64 and IA-32 Architectures Software
                        // Developer’s Manual, Volume 1: Basic Architecture":
                        //
                        // "Software enables the XSAVE feature set by setting
                        // CR4.OSXSAVE[bit 18] to 1 (e.g., with the MOV to CR4
                        // instruction). If this bit is 0, execution of any of XGETBV,
                        // XRSTOR, XRSTORS, XSAVE, XSAVEC, XSAVEOPT, XSAVES, and XSETBV
                        // causes an invalid-opcode exception (#UD)"
                        //
                        enable(proc_info_ecx, 26, Feature::xsave);

                        // For `xsaveopt`, `xsavec`, and `xsaves` we need to query:
                        // Processor Extended State Enumeration Sub-leaf (EAX = 0DH,
                        // ECX = 1):
                        if max_basic_leaf >= 0xd {
                            let CpuidResult {
                                eax: proc_extended_state1_eax,
                                ..
                            } = unsafe { __cpuid_count(0xd_u32, 1) };
                            enable(proc_extended_state1_eax, 0, Feature::xsaveopt);
                            enable(proc_extended_state1_eax, 1, Feature::xsavec);
                            enable(proc_extended_state1_eax, 3, Feature::xsaves);
                        }

                        // FMA (uses 256-bit wide registers):
                        enable(proc_info_ecx, 12, Feature::fma);

                        // And AVX/AVX2:
                        enable(proc_info_ecx, 28, Feature::avx);
                        enable(extended_features_ebx, 5, Feature::avx2);

                        // For AVX-512 the OS also needs to support saving/restoring
                        // the extended state, only then we enable AVX-512 support:
                        if os_avx512_support {
                            enable(extended_features_ebx, 16, Feature::avx512f);
                            enable(extended_features_ebx, 17, Feature::avx512dq);
                            enable(extended_features_ebx, 21, Feature::avx512_ifma);
                            enable(extended_features_ebx, 26, Feature::avx512pf);
                            enable(extended_features_ebx, 27, Feature::avx512er);
                            enable(extended_features_ebx, 28, Feature::avx512cd);
                            enable(extended_features_ebx, 30, Feature::avx512bw);
                            enable(extended_features_ebx, 31, Feature::avx512vl);
                            enable(extended_features_ecx, 1, Feature::avx512_vbmi);
                            enable(
                                extended_features_ecx,
                                14,
                                Feature::avx512_vpopcntdq,
                            );
                        }
                    }
                }
            }

            // This detects ABM on AMD CPUs and LZCNT on Intel CPUs.
            // On intel CPUs with popcnt, lzcnt implements the
            // "missing part" of ABM, so we map both to the same
            // internal feature.
            //
            // The `is_x86_feature_detected!("lzcnt")` macro then
            // internally maps to Feature::abm.
            enable(extended_proc_info_ecx, 5, Feature::abm);
            if vendor_id == *b"AuthenticAMD" {
                // These features are only available on AMD CPUs:
                enable(extended_proc_info_ecx, 6, Feature::sse4a);
                enable(extended_proc_info_ecx, 21, Feature::tbm);
            }
        }

        value
    }

    #[cfg(test)]
    mod tests {
        extern crate cupid;

        #[test]
        fn dump() {
            println!("aes: {:?}", is_x86_feature_detected!("aes"));
            println!("pclmulqdq: {:?}", is_x86_feature_detected!("pclmulqdq"));
            println!("rdrand: {:?}", is_x86_feature_detected!("rdrand"));
            println!("rdseed: {:?}", is_x86_feature_detected!("rdseed"));
            println!("tsc: {:?}", is_x86_feature_detected!("tsc"));
            println!("sse: {:?}", is_x86_feature_detected!("sse"));
            println!("sse2: {:?}", is_x86_feature_detected!("sse2"));
            println!("sse3: {:?}", is_x86_feature_detected!("sse3"));
            println!("ssse3: {:?}", is_x86_feature_detected!("ssse3"));
            println!("sse4.1: {:?}", is_x86_feature_detected!("sse4.1"));
            println!("sse4.2: {:?}", is_x86_feature_detected!("sse4.2"));
            println!("sse4a: {:?}", is_x86_feature_detected!("sse4a"));
            println!("sha: {:?}", is_x86_feature_detected!("sha"));
            println!("avx: {:?}", is_x86_feature_detected!("avx"));
            println!("avx2: {:?}", is_x86_feature_detected!("avx2"));
            println!("avx512f {:?}", is_x86_feature_detected!("avx512f"));
            println!("avx512cd {:?}", is_x86_feature_detected!("avx512cd"));
            println!("avx512er {:?}", is_x86_feature_detected!("avx512er"));
            println!("avx512pf {:?}", is_x86_feature_detected!("avx512pf"));
            println!("avx512bw {:?}", is_x86_feature_detected!("avx512bw"));
            println!("avx512dq {:?}", is_x86_feature_detected!("avx512dq"));
            println!("avx512vl {:?}", is_x86_feature_detected!("avx512vl"));
            println!("avx512_ifma {:?}", is_x86_feature_detected!("avx512ifma"));
            println!("avx512_vbmi {:?}", is_x86_feature_detected!("avx512vbmi"));
            println!(
                "avx512_vpopcntdq {:?}",
                is_x86_feature_detected!("avx512vpopcntdq")
            );
            println!("fma: {:?}", is_x86_feature_detected!("fma"));
            println!("abm: {:?}", is_x86_feature_detected!("abm"));
            println!("bmi: {:?}", is_x86_feature_detected!("bmi1"));
            println!("bmi2: {:?}", is_x86_feature_detected!("bmi2"));
            println!("tbm: {:?}", is_x86_feature_detected!("tbm"));
            println!("popcnt: {:?}", is_x86_feature_detected!("popcnt"));
            println!("lzcnt: {:?}", is_x86_feature_detected!("lzcnt"));
            println!("fxsr: {:?}", is_x86_feature_detected!("fxsr"));
            println!("xsave: {:?}", is_x86_feature_detected!("xsave"));
            println!("xsaveopt: {:?}", is_x86_feature_detected!("xsaveopt"));
            println!("xsaves: {:?}", is_x86_feature_detected!("xsaves"));
            println!("xsavec: {:?}", is_x86_feature_detected!("xsavec"));
        }

        #[test]
        fn compare_with_cupid() {
            let information = cupid::master().unwrap();
            assert_eq!(is_x86_feature_detected!("aes"), information.aesni());
            assert_eq!(is_x86_feature_detected!("pclmulqdq"), information.pclmulqdq());
            assert_eq!(is_x86_feature_detected!("rdrand"), information.rdrand());
            assert_eq!(is_x86_feature_detected!("rdseed"), information.rdseed());
            assert_eq!(is_x86_feature_detected!("tsc"), information.tsc());
            assert_eq!(is_x86_feature_detected!("sse"), information.sse());
            assert_eq!(is_x86_feature_detected!("sse2"), information.sse2());
            assert_eq!(is_x86_feature_detected!("sse3"), information.sse3());
            assert_eq!(is_x86_feature_detected!("ssse3"), information.ssse3());
            assert_eq!(is_x86_feature_detected!("sse4.1"), information.sse4_1());
            assert_eq!(is_x86_feature_detected!("sse4.2"), information.sse4_2());
            assert_eq!(is_x86_feature_detected!("sse4a"), information.sse4a());
            assert_eq!(is_x86_feature_detected!("sha"), information.sha());
            assert_eq!(is_x86_feature_detected!("avx"), information.avx());
            assert_eq!(is_x86_feature_detected!("avx2"), information.avx2());
            assert_eq!(is_x86_feature_detected!("avx512f"), information.avx512f());
            assert_eq!(is_x86_feature_detected!("avx512cd"), information.avx512cd());
            assert_eq!(is_x86_feature_detected!("avx512er"), information.avx512er());
            assert_eq!(is_x86_feature_detected!("avx512pf"), information.avx512pf());
            assert_eq!(is_x86_feature_detected!("avx512bw"), information.avx512bw());
            assert_eq!(is_x86_feature_detected!("avx512dq"), information.avx512dq());
            assert_eq!(is_x86_feature_detected!("avx512vl"), information.avx512vl());
            assert_eq!(
                is_x86_feature_detected!("avx512ifma"),
                information.avx512_ifma()
            );
            assert_eq!(
                is_x86_feature_detected!("avx512vbmi"),
                information.avx512_vbmi()
            );
            assert_eq!(
                is_x86_feature_detected!("avx512vpopcntdq"),
                information.avx512_vpopcntdq()
            );
            assert_eq!(is_x86_feature_detected!("fma"), information.fma());
            assert_eq!(is_x86_feature_detected!("bmi1"), information.bmi1());
            assert_eq!(is_x86_feature_detected!("bmi2"), information.bmi2());
            assert_eq!(is_x86_feature_detected!("popcnt"), information.popcnt());
            assert_eq!(is_x86_feature_detected!("abm"), information.lzcnt());
            assert_eq!(is_x86_feature_detected!("tbm"), information.tbm());
            assert_eq!(is_x86_feature_detected!("lzcnt"), information.lzcnt());
            assert_eq!(is_x86_feature_detected!("xsave"), information.xsave());
            assert_eq!(is_x86_feature_detected!("xsaveopt"), information.xsaveopt());
            assert_eq!(
                is_x86_feature_detected!("xsavec"),
                information.xsavec_and_xrstor()
            );
            assert_eq!(
                is_x86_feature_detected!("xsaves"),
                information.xsaves_xrstors_and_ia32_xss()
            );
        }
    }

# Help

## Keyboard Shortcuts

`?`
  Show this help dialog
`S`
  Focus the search field
`↑`
  Move up in search results
`↓`
  Move down in search results
`↹`
  Switch tab
`⏎`
  Go to active search result
`+`
  Expand all sections
`-`
  Collapse all sections

## Search Tricks

Prefix searches with a type followed by a colon (e.g. `fn:`) to restrict the search to a given type.

Accepted types are: `fn`, `mod`, `struct`, `enum`, `trait`, `type`, `macro`, and `const`.

Search functions by type signature (e.g. `vec -> usize` or `* -> vec`)

Search multiple things at once by splitting your query with comma (e.g. `str,u8` or `String,struct:Vec,test`)