# HG changeset patch # User Robert Mustacchi # Date 1524241329 0 # Node ID ddf0566cb4f43ab48c9e49beb7e2b7b0e25df171 # Parent 628a13d9b7eb6dd2ddf10b0d86b922622d11c410 9747 Implement CPU autoreplace based on Intel PPIN Reviewed by: Jerry Jelinek Reviewed by: Rob Johnston Reviewed by: Richard Lowe Approved by: Dan McDonald diff -r 628a13d9b7eb -r ddf0566cb4f4 usr/src/lib/fm/topo/modules/i86pc/chip/chip.c --- a/usr/src/lib/fm/topo/modules/i86pc/chip/chip.c Thu Nov 15 10:17:46 2018 +0000 +++ b/usr/src/lib/fm/topo/modules/i86pc/chip/chip.c Fri Apr 20 16:22:09 2018 +0000 @@ -22,6 +22,7 @@ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. + * Copyright (c) 2018, Joyent, Inc. */ #include @@ -86,6 +87,9 @@ TOPO_STABILITY_INTERNAL, a4fplus_chip_label}, { FSB2_CHIP_LBL, "Property method", 0, TOPO_STABILITY_INTERNAL, fsb2_chip_label}, + { TOPO_METH_REPLACED, TOPO_METH_REPLACED_DESC, + TOPO_METH_REPLACED_VERSION, TOPO_STABILITY_INTERNAL, + chip_fmri_replaced }, { NULL } }; @@ -143,7 +147,7 @@ static tnode_t * create_node(topo_mod_t *mod, tnode_t *pnode, nvlist_t *auth, char *name, - topo_instance_t inst, uint16_t smbios_id) + topo_instance_t inst, nvlist_t *cpu, uint16_t smbios_id) { nvlist_t *fmri; tnode_t *cnode; @@ -179,6 +183,17 @@ topo_mod_strfree(mod, (char *)serial); topo_mod_strfree(mod, (char *)part); topo_mod_strfree(mod, (char *)rev); + } else { + char *serial = NULL; + + if (nvlist_lookup_string(cpu, FM_PHYSCPU_INFO_CHIP_IDENTSTR, + &serial) == 0) { + if (nvlist_add_string(fmri, FM_FMRI_HC_SERIAL_ID, + serial) != 0) { + whinge(mod, NULL, + "create_node: nvlist_add_string failed\n"); + } + } } cnode = topo_node_bind(mod, pnode, name, inst, fmri); @@ -218,7 +233,7 @@ } if ((strand = create_node(mod, pnode, auth, STRAND_NODE_NAME, - strandid, chip_smbiosid)) == NULL) + strandid, cpu, chip_smbiosid)) == NULL) return (-1); /* @@ -339,7 +354,7 @@ } if ((core = topo_node_lookup(pnode, CORE_NODE_NAME, coreid)) == NULL) { if ((core = create_node(mod, pnode, auth, CORE_NODE_NAME, - coreid, chip_smbiosid)) == NULL) + coreid, cpu, chip_smbiosid)) == NULL) return (-1); /* @@ -508,7 +523,7 @@ if ((chip = topo_node_lookup(pnode, CHIP_NODE_NAME, chipid)) == NULL) { if ((chip = create_node(mod, pnode, auth, CHIP_NODE_NAME, - chipid, smbios_id)) == NULL) + chipid, cpu, smbios_id)) == NULL) return (-1); /* * Do not register XML map methods if SMBIOS can provide diff -r 628a13d9b7eb -r ddf0566cb4f4 usr/src/lib/fm/topo/modules/i86pc/chip/chip.h --- a/usr/src/lib/fm/topo/modules/i86pc/chip/chip.h Thu Nov 15 10:17:46 2018 +0000 +++ b/usr/src/lib/fm/topo/modules/i86pc/chip/chip.h Fri Apr 20 16:22:09 2018 +0000 @@ -20,6 +20,7 @@ */ /* * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, Joyent, Inc. */ #ifndef _CHIP_H @@ -182,6 +183,8 @@ nvlist_t *, nvlist_t **); extern int ntv_page_unusable(topo_mod_t *, tnode_t *, topo_version_t, nvlist_t *, nvlist_t **); +extern int chip_fmri_replaced(topo_mod_t *, tnode_t *, topo_version_t, + nvlist_t *, nvlist_t **); extern int mem_asru_create(topo_mod_t *, nvlist_t *, nvlist_t **); diff -r 628a13d9b7eb -r ddf0566cb4f4 usr/src/lib/fm/topo/modules/i86pc/chip/chip_subr.c --- a/usr/src/lib/fm/topo/modules/i86pc/chip/chip_subr.c Thu Nov 15 10:17:46 2018 +0000 +++ b/usr/src/lib/fm/topo/modules/i86pc/chip/chip_subr.c Fri Apr 20 16:22:09 2018 +0000 @@ -22,6 +22,7 @@ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. + * Copyright (c) 2018, Joyent, Inc. */ /* @@ -812,3 +813,54 @@ return (set_retnvl(mod, out, TOPO_METH_UNUSABLE_RET, rc == FMD_AGENT_RETIRE_DONE ? 1 : 0)); } + +/* + * Determine whether or not we believe a chip has been replaced. While it's + * tempting to just do a straight up comparison of the FMRI and its serial + * number, things are not that straightforward. + * + * The presence of a serial number on the CPU is not always guaranteed. It is + * possible that systems firmware can hide the information required to generate + * a synthesized serial number or that it is strictly not present. As such, we + * will only declare something replaced when both the old and current resource + * have a serial number present. If it is missing for whatever reason, then we + * cannot assume anything about a replacement having occurred. + * + * This logic applies regardless of whether or not we have an FM-aware SMBIOS. + */ +int +chip_fmri_replaced(topo_mod_t *mod, tnode_t *node, topo_version_t version, + nvlist_t *in, nvlist_t **out) +{ + nvlist_t *rsrc = NULL; + int err, ret; + char *old_serial, *new_serial; + + if (version > TOPO_METH_REPLACED_VERSION) + return (topo_mod_seterrno(mod, EMOD_VER_NEW)); + + if (topo_node_resource(node, &rsrc, &err) == -1) { + return (topo_mod_seterrno(mod, err)); + } + + if (nvlist_lookup_string(rsrc, FM_FMRI_HC_SERIAL_ID, + &new_serial) != 0) { + ret = FMD_OBJ_STATE_UNKNOWN; + goto out; + } + + if (nvlist_lookup_string(in, FM_FMRI_HC_SERIAL_ID, &old_serial) != 0) { + ret = FMD_OBJ_STATE_UNKNOWN; + goto out; + } + + if (strcmp(old_serial, new_serial) == 0) { + ret = FMD_OBJ_STATE_STILL_PRESENT; + } else { + ret = FMD_OBJ_STATE_REPLACED; + } + +out: + nvlist_free(rsrc); + return (set_retnvl(mod, out, TOPO_METH_REPLACED_RET, ret)); +} diff -r 628a13d9b7eb -r ddf0566cb4f4 usr/src/uts/common/sys/devfm.h --- a/usr/src/uts/common/sys/devfm.h Thu Nov 15 10:17:46 2018 +0000 +++ b/usr/src/uts/common/sys/devfm.h Fri Apr 20 16:22:09 2018 +0000 @@ -21,6 +21,7 @@ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. + * Copyright (c) 2018, Joyent, Inc. */ #ifndef _SYS_DEVFM_H @@ -120,6 +121,7 @@ #define FM_PHYSCPU_INFO_CHIP_REV "chip_rev" #define FM_PHYSCPU_INFO_SOCKET_TYPE "socket_type" #define FM_PHYSCPU_INFO_CPU_ID "cpuid" +#define FM_PHYSCPU_INFO_CHIP_IDENTSTR "chip_identstr" #ifdef __cplusplus } diff -r 628a13d9b7eb -r ddf0566cb4f4 usr/src/uts/i86pc/cpu/generic_cpu/gcpu.h --- a/usr/src/uts/i86pc/cpu/generic_cpu/gcpu.h Thu Nov 15 10:17:46 2018 +0000 +++ b/usr/src/uts/i86pc/cpu/generic_cpu/gcpu.h Fri Apr 20 16:22:09 2018 +0000 @@ -190,6 +190,7 @@ kmutex_t gcpus_poll_lock; /* serialize pollers on the same chip */ uint32_t gcpus_actv_banks; /* MCA bank numbers active on chip */ volatile uint32_t gcpus_actv_cnt; /* active cpu count in this chip */ + char *gcpus_ident; /* ident string, if available */ }; struct gcpu_data { diff -r 628a13d9b7eb -r ddf0566cb4f4 usr/src/uts/i86pc/cpu/generic_cpu/gcpu_main.c --- a/usr/src/uts/i86pc/cpu/generic_cpu/gcpu_main.c Thu Nov 15 10:17:46 2018 +0000 +++ b/usr/src/uts/i86pc/cpu/generic_cpu/gcpu_main.c Fri Apr 20 16:22:09 2018 +0000 @@ -42,6 +42,7 @@ #include #include #include +#include #include "gcpu.h" @@ -52,6 +53,115 @@ #define GCPU_MAX_CHIPID 32 static struct gcpu_chipshared *gcpu_shared[GCPU_MAX_CHIPID]; +#ifdef DEBUG +int gcpu_id_disable = 0; +static const char *gcpu_id_override[GCPU_MAX_CHIPID] = { NULL }; +#endif + +#ifndef __xpv +/* + * This should probably be delegated to a CPU specific module. However, as those + * haven't been developed as actively for recent CPUs, we should revisit this + * when we do have it and move this out of gcpu. + * + * This method is only supported on Intel Xeon platforms. It relies on a + * combination of the PPIN and the cpuid signature. Both are required to form + * the synthetic ID. This ID is preceded with iv0-INTC to represent that this is + * an Intel synthetic ID. The iv0 is the illumos version zero of the ID for + * Intel. If we have a new scheme for a new generation of processors, then that + * should rev the version field, otherwise for a given processor, this synthetic + * ID should not change. For more information on PPIN and these MSRS, see the + * relevant processor external design specification. + */ +static char * +gcpu_init_ident_intc(cmi_hdl_t hdl) +{ + uint64_t msr; + + /* + * This list should be extended as new Intel Xeon family processors come + * out. + */ + switch (cmi_hdl_model(hdl)) { + case INTC_MODEL_IVYBRIDGE_XEON: + case INTC_MODEL_HASWELL_XEON: + case INTC_MODEL_BROADWELL_XEON: + case INTC_MODEL_BROADWELL_XEON_D: + case INTC_MODEL_SKYLAKE_XEON: + break; + default: + return (NULL); + } + + if (cmi_hdl_rdmsr(hdl, MSR_PLATFORM_INFO, &msr) != CMI_SUCCESS) { + return (NULL); + } + + if ((msr & MSR_PLATFORM_INFO_PPIN) == 0) { + return (NULL); + } + + if (cmi_hdl_rdmsr(hdl, MSR_PPIN_CTL, &msr) != CMI_SUCCESS) { + return (NULL); + } + + if ((msr & MSR_PPIN_CTL_ENABLED) == 0) { + if ((msr & MSR_PPIN_CTL_LOCKED) != 0) { + return (NULL); + } + + if (cmi_hdl_wrmsr(hdl, MSR_PPIN_CTL, MSR_PPIN_CTL_ENABLED) != + CMI_SUCCESS) { + return (NULL); + } + } + + if (cmi_hdl_rdmsr(hdl, MSR_PPIN, &msr) != CMI_SUCCESS) { + return (NULL); + } + + /* + * Now that we've read data, lock the PPIN. Don't worry about success or + * failure of this part, as we will have gotten everything that we need. + * It is possible that it locked open, for example. + */ + (void) cmi_hdl_wrmsr(hdl, MSR_PPIN_CTL, MSR_PPIN_CTL_LOCKED); + + return (kmem_asprintf("iv0-INTC-%x-%llx", cmi_hdl_chipsig(hdl), msr)); +} +#endif /* __xpv */ + +static void +gcpu_init_ident(cmi_hdl_t hdl, struct gcpu_chipshared *sp) +{ +#ifdef DEBUG + uint_t chipid; + + /* + * On debug, allow a developer to override the string to more + * easily test CPU autoreplace without needing to physically + * replace a CPU. + */ + if (gcpu_id_disable != 0) { + return; + } + + chipid = cmi_hdl_chipid(hdl); + if (gcpu_id_override[chipid] != NULL) { + sp->gcpus_ident = strdup(gcpu_id_override[chipid]); + return; + } +#endif + +#ifndef __xpv + switch (cmi_hdl_vendor(hdl)) { + case X86_VENDOR_Intel: + sp->gcpus_ident = gcpu_init_ident_intc(hdl); + default: + break; + } +#endif /* __xpv */ +} /* * Our cmi_init entry point, called during startup of each cpu instance. @@ -90,6 +200,8 @@ mutex_destroy(&sp->gcpus_poll_lock); kmem_free(sp, sizeof (struct gcpu_chipshared)); sp = osp; + } else { + gcpu_init_ident(hdl, sp); } } @@ -165,6 +277,26 @@ #endif } +const char * +gcpu_ident(cmi_hdl_t hdl) +{ + uint_t chipid; + struct gcpu_chipshared *sp; + + if (gcpu_disable) + return (NULL); + + chipid = cmi_hdl_chipid(hdl); + if (chipid >= GCPU_MAX_CHIPID) + return (NULL); + + if (cmi_hdl_getcmidata(hdl) == NULL) + return (NULL); + + sp = gcpu_shared[cmi_hdl_chipid(hdl)]; + return (sp->gcpus_ident); +} + #ifdef __xpv #define GCPU_OP(ntvop, xpvop) xpvop #else @@ -186,6 +318,7 @@ GCPU_OP(gcpu_hdl_poke, NULL), /* cmi_hdl_poke */ gcpu_fini, /* cmi_fini */ GCPU_OP(NULL, gcpu_xpv_panic_callback), /* cmi_panic_callback */ + gcpu_ident /* cmi_ident */ }; static struct modlcpu modlcpu = { diff -r 628a13d9b7eb -r ddf0566cb4f4 usr/src/uts/i86pc/os/cmi.c --- a/usr/src/uts/i86pc/os/cmi.c Thu Nov 15 10:17:46 2018 +0000 +++ b/usr/src/uts/i86pc/os/cmi.c Fri Apr 20 16:22:09 2018 +0000 @@ -22,6 +22,7 @@ /* * Copyright 2010 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. + * Copyright (c) 2018, Joyent, Inc. */ /* @@ -967,3 +968,15 @@ cmi_hdl_rele(hdl); } + + +const char * +cmi_hdl_chipident(cmi_hdl_t hdl) +{ + cmi_t *cmi = cmi_hdl_getcmi(hdl); + + if (!CMI_OP_PRESENT(cmi, cmi_ident)) + return (NULL); + + return (CMI_OPS(cmi)->cmi_ident(hdl)); +} diff -r 628a13d9b7eb -r ddf0566cb4f4 usr/src/uts/i86pc/os/cmi_hw.c --- a/usr/src/uts/i86pc/os/cmi_hw.c Thu Nov 15 10:17:46 2018 +0000 +++ b/usr/src/uts/i86pc/os/cmi_hw.c Fri Apr 20 16:22:09 2018 +0000 @@ -21,6 +21,7 @@ /* * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, Joyent, Inc. */ /* * Copyright (c) 2010, Intel Corporation. @@ -121,6 +122,7 @@ const char *(*cmio_chiprevstr)(cmi_hdl_impl_t *); uint32_t (*cmio_getsockettype)(cmi_hdl_impl_t *); const char *(*cmio_getsocketstr)(cmi_hdl_impl_t *); + uint_t (*cmio_chipsig)(cmi_hdl_impl_t *); id_t (*cmio_logical_id)(cmi_hdl_impl_t *); /* @@ -684,6 +686,12 @@ return (cpuid_getsocketstr(HDLPRIV(hdl))); } +static uint_t +ntv_chipsig(cmi_hdl_impl_t *hdl) +{ + return (cpuid_getsig(HDLPRIV(hdl))); +} + static id_t ntv_logical_id(cmi_hdl_impl_t *hdl) { @@ -997,6 +1005,13 @@ xpv_model(hdl), xpv_stepping(hdl))); } +/* ARGSUSED */ +static uint_t +xpv_chipsig(cmi_hdl_impl_t *hdl) +{ + return (0); +} + static id_t xpv_logical_id(cmi_hdl_impl_t *hdl) { @@ -1595,6 +1610,7 @@ cmio_##what(IMPLHDL(ophdl))); \ } +/* BEGIN CSTYLED */ CMI_HDL_OPFUNC(vendor, uint_t) CMI_HDL_OPFUNC(vendorstr, const char *) CMI_HDL_OPFUNC(family, uint_t) @@ -1614,6 +1630,8 @@ CMI_HDL_OPFUNC(smbiosid, uint16_t) CMI_HDL_OPFUNC(smb_chipid, uint_t) CMI_HDL_OPFUNC(smb_bboard, nvlist_t *) +CMI_HDL_OPFUNC(chipsig, uint_t) +/* END CSTYLED */ boolean_t cmi_hdl_is_cmt(cmi_hdl_t ophdl) @@ -1990,6 +2008,7 @@ xpv_chiprevstr, /* cmio_chiprevstr */ xpv_getsockettype, /* cmio_getsockettype */ xpv_getsocketstr, /* cmio_getsocketstr */ + xpv_chipsig, /* cmio_chipsig */ xpv_logical_id, /* cmio_logical_id */ NULL, /* cmio_getcr4 */ NULL, /* cmio_setcr4 */ @@ -2022,6 +2041,7 @@ ntv_chiprevstr, /* cmio_chiprevstr */ ntv_getsockettype, /* cmio_getsockettype */ ntv_getsocketstr, /* cmio_getsocketstr */ + ntv_chipsig, /* cmio_chipsig */ ntv_logical_id, /* cmio_logical_id */ ntv_getcr4, /* cmio_getcr4 */ ntv_setcr4, /* cmio_setcr4 */ diff -r 628a13d9b7eb -r ddf0566cb4f4 usr/src/uts/i86pc/sys/cpu_module_impl.h --- a/usr/src/uts/i86pc/sys/cpu_module_impl.h Thu Nov 15 10:17:46 2018 +0000 +++ b/usr/src/uts/i86pc/sys/cpu_module_impl.h Fri Apr 20 16:22:09 2018 +0000 @@ -22,6 +22,7 @@ /* * Copyright 2008 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. + * Copyright (c) 2018, Joyent, Inc. */ #ifndef _SYS_CPU_MODULE_IMPL_H @@ -64,6 +65,7 @@ void (*cmi_hdl_poke)(cmi_hdl_t); void (*cmi_fini)(cmi_hdl_t); void (*cmi_panic_callback)(void); + const char *(*cmi_ident)(cmi_hdl_t); } cmi_ops_t; /* diff -r 628a13d9b7eb -r ddf0566cb4f4 usr/src/uts/intel/io/devfm_machdep.c --- a/usr/src/uts/intel/io/devfm_machdep.c Thu Nov 15 10:17:46 2018 +0000 +++ b/usr/src/uts/intel/io/devfm_machdep.c Fri Apr 20 16:22:09 2018 +0000 @@ -21,6 +21,7 @@ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. + * Copyright (c) 2018, Joyent, Inc. */ #include @@ -150,6 +151,7 @@ { uint_t fm_chipid; uint16_t smbios_id; + const char *idstr; (void) nvlist_alloc(nvlp, NV_UNIQUE_NAME, KM_SLEEP); @@ -203,6 +205,16 @@ FM_PHYSCPU_INFO_CPU_ID, DATA_TYPE_INT32, (int32_t)cmi_hdl_logical_id(hdl), NULL); + + /* + * Do this separately so that way if there is no ident string we do not + * trigger an error. + */ + if ((idstr = cmi_hdl_chipident(hdl)) != NULL) { + fm_payload_set(*nvlp, + FM_PHYSCPU_INFO_CHIP_IDENTSTR, DATA_TYPE_STRING, idstr, + NULL); + } } /*ARGSUSED*/ diff -r 628a13d9b7eb -r ddf0566cb4f4 usr/src/uts/intel/sys/cpu_module.h --- a/usr/src/uts/intel/sys/cpu_module.h Thu Nov 15 10:17:46 2018 +0000 +++ b/usr/src/uts/intel/sys/cpu_module.h Fri Apr 20 16:22:09 2018 +0000 @@ -22,6 +22,7 @@ /* * Copyright 2010 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. + * Copyright (c) 2018, Joyent, Inc. */ #ifndef _SYS_CPU_MODULE_H @@ -163,6 +164,8 @@ extern uint16_t cmi_hdl_smbiosid(cmi_hdl_t); extern uint_t cmi_hdl_smb_chipid(cmi_hdl_t); extern nvlist_t *cmi_hdl_smb_bboard(cmi_hdl_t); +extern uint_t cmi_hdl_chipsig(cmi_hdl_t); +extern const char *cmi_hdl_chipident(cmi_hdl_t); extern int cmi_hdl_online(cmi_hdl_t, int, int *); diff -r 628a13d9b7eb -r ddf0566cb4f4 usr/src/uts/intel/sys/x86_archext.h --- a/usr/src/uts/intel/sys/x86_archext.h Thu Nov 15 10:17:46 2018 +0000 +++ b/usr/src/uts/intel/sys/x86_archext.h Fri Apr 20 16:22:09 2018 +0000 @@ -248,7 +248,7 @@ #define CPUID_INTC_ECX_7_0_AVX512VPOPCDQ 0x00004000 /* AVX512 VPOPCNTDQ */ #define CPUID_INTC_ECX_7_0_ALL_AVX512 \ - (CPUID_INTC_ECX_7_0_AVX512VBMI | CPUID_INTC_ECX_7_0_AVX512VPOPCDQ) +(CPUID_INTC_ECX_7_0_AVX512VBMI | CPUID_INTC_ECX_7_0_AVX512VPOPCDQ) #define CPUID_INTC_EDX_7_0_AVX5124NNIW 0x00000004 /* AVX512 4NNIW */ #define CPUID_INTC_EDX_7_0_AVX5124FMAPS 0x00000008 /* AVX512 4FMAPS */ @@ -348,6 +348,18 @@ #define MSR_PRP4_LBSTK_TO_15 0x6cf /* + * General Xeon based MSRs + */ +#define MSR_PPIN_CTL 0x04e +#define MSR_PPIN 0x04f +#define MSR_PLATFORM_INFO 0x0ce + +#define MSR_PLATFORM_INFO_PPIN (1 << 23) +#define MSR_PPIN_CTL_MASK 0x03 +#define MSR_PPIN_CTL_LOCKED 0x01 +#define MSR_PPIN_CTL_ENABLED 0x02 + +/* * Intel IA32_ARCH_CAPABILITIES MSR. */ #define MSR_IA32_ARCH_CAPABILITIES 0x10a @@ -712,6 +724,19 @@ #define X86_SOCKET_FS1R2 _X86_SOCKET_MKVAL(X86_VENDOR_AMD, 0x100000) #define X86_SOCKET_FM2 _X86_SOCKET_MKVAL(X86_VENDOR_AMD, 0x200000) + +/* + * Definitions for Intel processor models. Note, these model values can overlap + * in a given family. Processor models are added here on an as needed basis. The + * Xeon extension here is to refer to what has been called the EP/EX lines or + * E5/E7, generally multi-socket capable processors. + */ +#define INTC_MODEL_IVYBRIDGE_XEON 0x3E +#define INTC_MODEL_HASWELL_XEON 0x3F +#define INTC_MODEL_BROADWELL_XEON 0x4F +#define INTC_MODEL_BROADWELL_XEON_D 0x56 +#define INTC_MODEL_SKYLAKE_XEON 0x55 + /* * xgetbv/xsetbv support * See section 13.3 in vol. 1 of the Intel devlopers manual.