detectNativeFeatures

fn detectNativeFeatures(cpu: *Target.Cpu, os_tag: Target.Os.Tag) void

File

Code

fn detectNativeFeatures(cpu: *Target.Cpu, os_tag: Target.Os.Tag) void {
    var leaf = cpuid(0, 0);

    const max_level = leaf.eax;

    leaf = cpuid(1, 0);

    setFeature(cpu, .sse3, bit(leaf.ecx, 0));
    setFeature(cpu, .pclmul, bit(leaf.ecx, 1));
    setFeature(cpu, .ssse3, bit(leaf.ecx, 9));
    setFeature(cpu, .cx16, bit(leaf.ecx, 13));
    setFeature(cpu, .sse4_1, bit(leaf.ecx, 19));
    setFeature(cpu, .sse4_2, bit(leaf.ecx, 20));
    setFeature(cpu, .movbe, bit(leaf.ecx, 22));
    setFeature(cpu, .popcnt, bit(leaf.ecx, 23));
    setFeature(cpu, .aes, bit(leaf.ecx, 25));
    setFeature(cpu, .rdrnd, bit(leaf.ecx, 30));

    setFeature(cpu, .cx8, bit(leaf.edx, 8));
    setFeature(cpu, .cmov, bit(leaf.edx, 15));
    setFeature(cpu, .mmx, bit(leaf.edx, 23));
    setFeature(cpu, .fxsr, bit(leaf.edx, 24));
    setFeature(cpu, .sse, bit(leaf.edx, 25));
    setFeature(cpu, .sse2, bit(leaf.edx, 26));

    const has_xsave = bit(leaf.ecx, 27);
    const has_avx = bit(leaf.ecx, 28);

    // Make sure not to call xgetbv if xsave is not supported
    const xcr0: Xcr0 = if (has_xsave and has_avx) @bitCast(getXCR0()) else @bitCast(@as(u32, 0));

    const has_avx_save = xcr0.sse and xcr0.avx;

    // LLVM approaches avx512_save by hardcoding it to true on Darwin,
    // because the kernel saves the context even if the bit is not set.
    // https://github.com/llvm/llvm-project/blob/bca373f73fc82728a8335e7d6cd164e8747139ec/llvm/lib/Support/Host.cpp#L1378
    //
    // Google approaches this by using a different series of checks and flags,
    // and this may report the feature more accurately on a technically correct
    // but ultimately less useful level.
    // https://github.com/google/cpu_features/blob/b5c271c53759b2b15ff91df19bd0b32f2966e275/src/cpuinfo_x86.c#L113
    // (called from https://github.com/google/cpu_features/blob/b5c271c53759b2b15ff91df19bd0b32f2966e275/src/cpuinfo_x86.c#L1052)
    //
    // Right now, we use LLVM's approach, because even if the target doesn't support
    // the feature, the kernel should provide the same functionality transparently,
    // so the implementation details don't make a difference.
    // That said, this flag impacts other CPU features' availability,
    // so until we can verify that this doesn't come with side affects,
    // we'll say TODO verify this.

    // Darwin lazily saves the AVX512 context on first use: trust that the OS will
    // save the AVX512 context if we use AVX512 instructions, even if the bit is not
    // set right now.
    const has_avx512_save = if (os_tag.isDarwin())
        true
    else
        xcr0.zmm_hi256 and xcr0.hi16_zmm;

    // AMX requires additional context to be saved by the OS.
    const has_amx_save = xcr0.xtilecfg and xcr0.xtiledata;

    const has_apx_save = xcr0.apx;

    setFeature(cpu, .avx, has_avx_save);
    setFeature(cpu, .fma, bit(leaf.ecx, 12) and has_avx_save);
    // Only enable XSAVE if OS has enabled support for saving YMM state.
    setFeature(cpu, .xsave, bit(leaf.ecx, 26) and has_avx_save);
    setFeature(cpu, .f16c, bit(leaf.ecx, 29) and has_avx_save);

    leaf = cpuid(0x80000000, 0);
    const max_ext_level = leaf.eax;

    if (max_ext_level >= 0x80000001) {
        leaf = cpuid(0x80000001, 0);

        setFeature(cpu, .sahf, bit(leaf.ecx, 0));
        setFeature(cpu, .lzcnt, bit(leaf.ecx, 5));
        setFeature(cpu, .sse4a, bit(leaf.ecx, 6));
        setFeature(cpu, .prfchw, bit(leaf.ecx, 8));
        setFeature(cpu, .xop, bit(leaf.ecx, 11) and has_avx_save);
        setFeature(cpu, .lwp, bit(leaf.ecx, 15));
        setFeature(cpu, .fma4, bit(leaf.ecx, 16) and has_avx_save);
        setFeature(cpu, .tbm, bit(leaf.ecx, 21));
        setFeature(cpu, .mwaitx, bit(leaf.ecx, 29));

        setFeature(cpu, .@"64bit", bit(leaf.edx, 29));
    } else {
        for ([_]Target.x86.Feature{
            .sahf,
            .lzcnt,
            .sse4a,
            .prfchw,
            .xop,
            .lwp,
            .fma4,
            .tbm,
            .mwaitx,

            .@"64bit",
        }) |feat| {
            setFeature(cpu, feat, false);
        }
    }

    // Misc. memory-related features.
    if (max_ext_level >= 0x80000008) {
        leaf = cpuid(0x80000008, 0);

        setFeature(cpu, .clzero, bit(leaf.ebx, 0));
        setFeature(cpu, .rdpru, bit(leaf.ebx, 4));
        setFeature(cpu, .wbnoinvd, bit(leaf.ebx, 9));
    } else {
        for ([_]Target.x86.Feature{
            .clzero,
            .rdpru,
            .wbnoinvd,
        }) |feat| {
            setFeature(cpu, feat, false);
        }
    }

    if (max_ext_level >= 0x80000021) {
        leaf = cpuid(0x80000021, 0);

        // AMD uses a different bit for prefetchi.
        setFeature(cpu, .prefetchi, bit(leaf.eax, 20));
    } else {
        for ([_]Target.x86.Feature{
            .prefetchi,
        }) |feat| {
            setFeature(cpu, feat, false);
        }
    }

    const has_avx10 = if (max_level >= 0x7) has_avx10: {
        leaf = cpuid(0x7, 0);

        setFeature(cpu, .fsgsbase, bit(leaf.ebx, 0));
        setFeature(cpu, .sgx, bit(leaf.ebx, 2));
        setFeature(cpu, .bmi, bit(leaf.ebx, 3));
        // AVX2 is only supported if we have the OS save support from AVX.
        setFeature(cpu, .avx2, bit(leaf.ebx, 5) and has_avx_save);
        setFeature(cpu, .smep, bit(leaf.ebx, 7));
        setFeature(cpu, .bmi2, bit(leaf.ebx, 8));
        setFeature(cpu, .invpcid, bit(leaf.ebx, 10));
        setFeature(cpu, .rtm, bit(leaf.ebx, 11));
        // AVX512 is only supported if the OS supports the context save for it.
        setFeature(cpu, .avx512f, bit(leaf.ebx, 16) and has_avx512_save);
        setFeature(cpu, .avx512dq, bit(leaf.ebx, 17) and has_avx512_save);
        setFeature(cpu, .rdseed, bit(leaf.ebx, 18));
        setFeature(cpu, .adx, bit(leaf.ebx, 19));
        setFeature(cpu, .smap, bit(leaf.ebx, 20));
        setFeature(cpu, .avx512ifma, bit(leaf.ebx, 21) and has_avx512_save);
        setFeature(cpu, .clflushopt, bit(leaf.ebx, 23));
        setFeature(cpu, .clwb, bit(leaf.ebx, 24));
        setFeature(cpu, .avx512pf, bit(leaf.ebx, 26) and has_avx512_save);
        setFeature(cpu, .avx512er, bit(leaf.ebx, 27) and has_avx512_save);
        setFeature(cpu, .avx512cd, bit(leaf.ebx, 28) and has_avx512_save);
        setFeature(cpu, .sha, bit(leaf.ebx, 29));
        setFeature(cpu, .avx512bw, bit(leaf.ebx, 30) and has_avx512_save);
        setFeature(cpu, .avx512vl, bit(leaf.ebx, 31) and has_avx512_save);

        setFeature(cpu, .prefetchwt1, bit(leaf.ecx, 0));
        setFeature(cpu, .avx512vbmi, bit(leaf.ecx, 1) and has_avx512_save);
        setFeature(cpu, .pku, bit(leaf.ecx, 4));
        setFeature(cpu, .waitpkg, bit(leaf.ecx, 5));
        setFeature(cpu, .avx512vbmi2, bit(leaf.ecx, 6) and has_avx512_save);
        setFeature(cpu, .shstk, bit(leaf.ecx, 7));
        setFeature(cpu, .gfni, bit(leaf.ecx, 8));
        setFeature(cpu, .vaes, bit(leaf.ecx, 9) and has_avx_save);
        setFeature(cpu, .vpclmulqdq, bit(leaf.ecx, 10) and has_avx_save);
        setFeature(cpu, .avx512vnni, bit(leaf.ecx, 11) and has_avx512_save);
        setFeature(cpu, .avx512bitalg, bit(leaf.ecx, 12) and has_avx512_save);
        setFeature(cpu, .avx512vpopcntdq, bit(leaf.ecx, 14) and has_avx512_save);
        setFeature(cpu, .rdpid, bit(leaf.ecx, 22));
        setFeature(cpu, .kl, bit(leaf.ecx, 23));
        setFeature(cpu, .cldemote, bit(leaf.ecx, 25));
        setFeature(cpu, .movdiri, bit(leaf.ecx, 27));
        setFeature(cpu, .movdir64b, bit(leaf.ecx, 28));
        setFeature(cpu, .enqcmd, bit(leaf.ecx, 29));

        // There are two CPUID leafs which information associated with the pconfig
        // instruction:
        // EAX=0x7, ECX=0x0 indicates the availability of the instruction (via the 18th
        // bit of EDX), while the EAX=0x1b leaf returns information on the
        // availability of specific pconfig leafs.
        // The target feature here only refers to the the first of these two.
        // Users might need to check for the availability of specific pconfig
        // leaves using cpuid, since that information is ignored while
        // detecting features using the "-march=native" flag.
        // For more info, see X86 ISA docs.
        setFeature(cpu, .uintr, bit(leaf.edx, 5));
        setFeature(cpu, .avx512vp2intersect, bit(leaf.edx, 8) and has_avx512_save);
        setFeature(cpu, .serialize, bit(leaf.edx, 14));
        setFeature(cpu, .tsxldtrk, bit(leaf.edx, 16));
        setFeature(cpu, .pconfig, bit(leaf.edx, 18));
        setFeature(cpu, .amx_bf16, bit(leaf.edx, 22) and has_amx_save);
        setFeature(cpu, .avx512fp16, bit(leaf.edx, 23) and has_avx512_save);
        setFeature(cpu, .amx_tile, bit(leaf.edx, 24) and has_amx_save);
        setFeature(cpu, .amx_int8, bit(leaf.edx, 25) and has_amx_save);

        if (leaf.eax >= 1) {
            leaf = cpuid(0x7, 0x1);

            setFeature(cpu, .sha512, bit(leaf.eax, 0));
            setFeature(cpu, .sm3, bit(leaf.eax, 1));
            setFeature(cpu, .sm4, bit(leaf.eax, 2));
            setFeature(cpu, .raoint, bit(leaf.eax, 3));
            setFeature(cpu, .avxvnni, bit(leaf.eax, 4) and has_avx_save);
            setFeature(cpu, .avx512bf16, bit(leaf.eax, 5) and has_avx512_save);
            setFeature(cpu, .cmpccxadd, bit(leaf.eax, 7));
            setFeature(cpu, .amx_fp16, bit(leaf.eax, 21) and has_amx_save);
            setFeature(cpu, .hreset, bit(leaf.eax, 22));
            setFeature(cpu, .avxifma, bit(leaf.eax, 23) and has_avx_save);

            setFeature(cpu, .avxvnniint8, bit(leaf.edx, 4) and has_avx_save);
            setFeature(cpu, .avxneconvert, bit(leaf.edx, 5) and has_avx_save);
            setFeature(cpu, .amx_complex, bit(leaf.edx, 8) and has_amx_save);
            setFeature(cpu, .avxvnniint16, bit(leaf.edx, 10) and has_avx_save);
            // This needs to account for prefetchi already being detected above on AMD.
            setFeature(cpu, .prefetchi, cpu.has(.x86, .prefetchi) or bit(leaf.edx, 14));
            setFeature(cpu, .usermsr, bit(leaf.edx, 15));
            // APX
            setFeature(cpu, .egpr, bit(leaf.edx, 21) and has_apx_save);
            setFeature(cpu, .push2pop2, bit(leaf.edx, 21) and has_apx_save);
            setFeature(cpu, .ppx, bit(leaf.edx, 21) and has_apx_save);
            setFeature(cpu, .ndd, bit(leaf.edx, 21) and has_apx_save);
            setFeature(cpu, .ccmp, bit(leaf.edx, 21) and has_apx_save);
            setFeature(cpu, .nf, bit(leaf.edx, 21) and has_apx_save);
            setFeature(cpu, .cf, bit(leaf.edx, 21) and has_apx_save);
            setFeature(cpu, .zu, bit(leaf.edx, 21) and has_apx_save);

            break :has_avx10 bit(leaf.edx, 19);
        } else {
            for ([_]Target.x86.Feature{
                .sha512,
                .sm3,
                .sm4,
                .raoint,
                .avxvnni,
                .avx512bf16,
                .cmpccxadd,
                .amx_fp16,
                .hreset,
                .avxifma,

                .avxvnniint8,
                .avxneconvert,
                .amx_complex,
                .avxvnniint16,
                // prefetchi already handled earlier.
                .usermsr,
                .egpr,
                .push2pop2,
                .ppx,
                .ndd,
                .ccmp,
                .nf,
                .cf,
                .zu,
            }) |feat| {
                setFeature(cpu, feat, false);
            }
        }

        break :has_avx10 false;
    } else has_avx10: {
        for ([_]Target.x86.Feature{
            .fsgsbase,
            .sgx,
            .bmi,
            .avx2,
            .smep,
            .bmi2,
            .invpcid,
            .rtm,
            .avx512f,
            .avx512dq,
            .rdseed,
            .adx,
            .smap,
            .avx512ifma,
            .clflushopt,
            .clwb,
            .avx512pf,
            .avx512er,
            .avx512cd,
            .sha,
            .avx512bw,
            .avx512vl,

            .prefetchwt1,
            .avx512vbmi,
            .pku,
            .waitpkg,
            .avx512vbmi2,
            .shstk,
            .gfni,
            .vaes,
            .vpclmulqdq,
            .avx512vnni,
            .avx512bitalg,
            .avx512vpopcntdq,
            .rdpid,
            .kl,
            .cldemote,
            .movdiri,
            .movdir64b,
            .enqcmd,

            .uintr,
            .avx512vp2intersect,
            .serialize,
            .tsxldtrk,
            .pconfig,
            .amx_bf16,
            .avx512fp16,
            .amx_tile,
            .amx_int8,

            .sha512,
            .sm3,
            .sm4,
            .raoint,
            .avxvnni,
            .avx512bf16,
            .cmpccxadd,
            .amx_fp16,
            .hreset,
            .avxifma,

            .avxvnniint8,
            .avxneconvert,
            .amx_complex,
            .avxvnniint16,
            // prefetchi already handled earlier.
            .usermsr,
            .egpr,
            .push2pop2,
            .ppx,
            .ndd,
            .ccmp,
            .nf,
            .cf,
            .zu,
        }) |feat| {
            setFeature(cpu, feat, false);
        }

        break :has_avx10 false;
    };

    if (max_level >= 0xD and has_avx_save) {
        leaf = cpuid(0xD, 0x1);

        // Only enable XSAVE if OS has enabled support for saving YMM state.
        setFeature(cpu, .xsaveopt, bit(leaf.eax, 0));
        setFeature(cpu, .xsavec, bit(leaf.eax, 1));
        setFeature(cpu, .xsaves, bit(leaf.eax, 3));
    } else {
        for ([_]Target.x86.Feature{
            .xsaveopt,
            .xsavec,
            .xsaves,
        }) |feat| {
            setFeature(cpu, feat, false);
        }
    }

    if (max_level >= 0x14) {
        leaf = cpuid(0x14, 0);

        setFeature(cpu, .ptwrite, bit(leaf.ebx, 4));
    } else {
        for ([_]Target.x86.Feature{
            .ptwrite,
        }) |feat| {
            setFeature(cpu, feat, false);
        }
    }

    if (max_level >= 0x19) {
        leaf = cpuid(0x19, 0);

        setFeature(cpu, .widekl, bit(leaf.ebx, 2));
    } else {
        for ([_]Target.x86.Feature{
            .widekl,
        }) |feat| {
            setFeature(cpu, feat, false);
        }
    }

    if (max_level >= 0x24) {
        leaf = cpuid(0x24, 0);

        const avx_ver = leaf.ebx & 0xff;

        setFeature(cpu, .avx10_1, has_avx10 and avx_ver >= 1);
        setFeature(cpu, .avx10_2, has_avx10 and avx_ver >= 2);
    } else {
        for ([_]Target.x86.Feature{
            .avx10_1,
            .avx10_2,
        }) |feat| {
            setFeature(cpu, feat, false);
        }
    }
}