diff options
Diffstat (limited to 'arch/sparc64')
56 files changed, 4013 insertions, 3688 deletions
diff --git a/arch/sparc64/Kconfig b/arch/sparc64/Kconfig index df3eacb5ca1..eb36f3b746b 100644 --- a/arch/sparc64/Kconfig +++ b/arch/sparc64/Kconfig @@ -1,9 +1,5 @@ -# $Id: config.in,v 1.158 2002/01/24 22:14:44 davem Exp $ -# For a description of the syntax of this configuration file, -# see the Configure script. -# - -mainmenu "Linux/UltraSPARC Kernel Configuration" +# sparc64 configuration +mainmenu "Linux Kernel Configuration for 64-bit SPARC" config SPARC bool @@ -17,12 +13,7 @@ config SPARC64 default y select HAVE_IDE select HAVE_LMB - help - SPARC is a family of RISC microprocessors designed and marketed by - Sun Microsystems, incorporated. This port covers the newer 64-bit - UltraSPARC. The UltraLinux project maintains both the SPARC32 and - SPARC64 ports; its web page is available at - <http://www.ultralinux.org/>. + select HAVE_ARCH_KGDB config GENERIC_TIME bool @@ -97,7 +88,7 @@ config SPARC64_PAGE_SIZE_8KB help This lets you select the page size of the kernel. - 8KB and 64KB work quite well, since Sparc ELF sections + 8KB and 64KB work quite well, since SPARC ELF sections provide for up to 64KB alignment. Therefore, 512KB and 4MB are for expert hackers only. @@ -138,7 +129,7 @@ config HOTPLUG_CPU bool "Support for hot-pluggable CPUs" depends on SMP select HOTPLUG - ---help--- + help Say Y here to experiment with turning CPUs off and on. CPUs can be controlled through /sys/devices/system/cpu/cpu#. Say N if you want to disable CPU hotplug. @@ -155,23 +146,16 @@ source "kernel/time/Kconfig" config SMP bool "Symmetric multi-processing support" - ---help--- + help This enables support for systems with more than one CPU. If you have a system with only one CPU, say N. If you have a system with more than one CPU, say Y. If you say N here, the kernel will run on single and multiprocessor machines, but will use only one CPU of a multiprocessor machine. If - you say Y here, the kernel will run on many, but not all, - singleprocessor machines. On a singleprocessor machine, the kernel - will run faster if you say N here. - - People using multiprocessor machines who say Y here should also say - Y to "Enhanced Real Time Clock Support", below. The "Advanced Power - Management" code will be disabled if you say Y here. - - See also <file:Documentation/nmi_watchdog.txt> and the SMP-HOWTO - available at <http://www.tldp.org/docs.html#howto>. + you say Y here, the kernel will run on single-processor machines. + On a single-processor machine, the kernel will run faster if you say + N here. If you don't know what to do here, say N. @@ -250,6 +234,26 @@ endchoice endmenu +config NUMA + bool "NUMA support" + +config NODES_SHIFT + int + default "4" + depends on NEED_MULTIPLE_NODES + +# Some NUMA nodes have memory ranges that span +# other nodes. Even though a pfn is valid and +# between a node's start and end pfns, it may not +# reside on that node. See memmap_init_zone() +# for details. +config NODES_SPAN_OTHER_NODES + def_bool y + depends on NEED_MULTIPLE_NODES + +config ARCH_POPULATES_NODE_MAP + def_bool y + config ARCH_SELECT_MEMORY_MODEL def_bool y @@ -264,50 +268,19 @@ source "mm/Kconfig" config ISA bool - help - Find out whether you have ISA slots on your motherboard. ISA is the - name of a bus system, i.e. the way the CPU talks to the other stuff - inside your box. Other bus systems are PCI, EISA, MicroChannel - (MCA) or VESA. ISA is an older system, now being displaced by PCI; - newer boards don't support it. If you have ISA, say Y, otherwise N. config ISAPNP bool - help - Say Y here if you would like support for ISA Plug and Play devices. - Some information is in <file:Documentation/isapnp.txt>. - - To compile this driver as a module, choose M here: the - module will be called isapnp. - - If unsure, say Y. config EISA bool - ---help--- - The Extended Industry Standard Architecture (EISA) bus was - developed as an open alternative to the IBM MicroChannel bus. - - The EISA bus provided some of the features of the IBM MicroChannel - bus while maintaining backward compatibility with cards made for - the older ISA bus. The EISA bus saw limited use between 1988 and - 1995 when it was made obsolete by the PCI bus. - - Say Y here if you are building a kernel for an EISA-based machine. - - Otherwise, say N. config MCA bool - help - MicroChannel Architecture is found in some IBM PS/2 machines and - laptops. It is a bus system similar to PCI or ISA. See - <file:Documentation/mca.txt> (and especially the web page given - there) before attempting to build an MCA bus kernel. config PCMCIA tristate - ---help--- + help Say Y here if you want to attach PCMCIA- or PC-cards to your Linux computer. These are credit-card size devices such as network cards, modems or hard drives often used with laptops computers. There are @@ -349,10 +322,10 @@ config PCI bool "PCI support" select ARCH_SUPPORTS_MSI help - Find out whether you have a PCI motherboard. PCI is the name of a - bus system, i.e. the way the CPU talks to the other stuff inside - your box. Other bus systems are ISA, EISA, MicroChannel (MCA) or - VESA. If you have PCI, say Y, otherwise N. + Find out whether your system includes a PCI bus. PCI is the name of + a bus system, i.e. the way the CPU talks to the other stuff inside + your box. If you say Y here, the kernel will include drivers and + infrastructure code to support PCI bus devices. config PCI_DOMAINS def_bool PCI @@ -376,15 +349,8 @@ menu "Executable file formats" source "fs/Kconfig.binfmt" -config SPARC32_COMPAT - bool "Kernel support for Linux/Sparc 32bit binary compatibility" - help - This allows you to run 32-bit binaries on your Ultra. - Everybody wants this; say Y. - config COMPAT bool - depends on SPARC32_COMPAT default y select COMPAT_BINFMT_ELF @@ -401,8 +367,8 @@ config SCHED_SMT default y help SMT scheduler support improves the CPU scheduler's decision making - when dealing with UltraSPARC cpus at a cost of slightly increased - overhead in some places. If unsure say N here. + when dealing with SPARC cpus at a cost of slightly increased overhead + in some places. If unsure say N here. config SCHED_MC bool "Multi-core scheduler support" diff --git a/arch/sparc64/defconfig b/arch/sparc64/defconfig index e1835868ad3..aff93c9d13f 100644 --- a/arch/sparc64/defconfig +++ b/arch/sparc64/defconfig @@ -1,7 +1,7 @@ # # Automatically generated make config: don't edit # Linux kernel version: 2.6.25 -# Sun Apr 20 01:33:21 2008 +# Sat Apr 26 03:11:06 2008 # CONFIG_SPARC=y CONFIG_SPARC64=y @@ -152,6 +152,10 @@ CONFIG_GENERIC_CALIBRATE_DELAY=y CONFIG_HUGETLB_PAGE_SIZE_4MB=y # CONFIG_HUGETLB_PAGE_SIZE_512K is not set # CONFIG_HUGETLB_PAGE_SIZE_64K is not set +CONFIG_NUMA=y +CONFIG_NODES_SHIFT=4 +CONFIG_NODES_SPAN_OTHER_NODES=y +CONFIG_ARCH_POPULATES_NODE_MAP=y CONFIG_ARCH_SELECT_MEMORY_MODEL=y CONFIG_ARCH_SPARSEMEM_ENABLE=y CONFIG_ARCH_SPARSEMEM_DEFAULT=y @@ -160,12 +164,14 @@ CONFIG_SELECT_MEMORY_MODEL=y # CONFIG_DISCONTIGMEM_MANUAL is not set CONFIG_SPARSEMEM_MANUAL=y CONFIG_SPARSEMEM=y +CONFIG_NEED_MULTIPLE_NODES=y CONFIG_HAVE_MEMORY_PRESENT=y # CONFIG_SPARSEMEM_STATIC is not set CONFIG_SPARSEMEM_EXTREME=y CONFIG_SPARSEMEM_VMEMMAP_ENABLE=y CONFIG_SPARSEMEM_VMEMMAP=y CONFIG_SPLIT_PTLOCK_CPUS=4 +CONFIG_MIGRATION=y CONFIG_RESOURCES_64BIT=y CONFIG_ZONE_DMA_FLAG=0 CONFIG_NR_QUICK=1 @@ -189,7 +195,6 @@ CONFIG_SUN_OPENPROMFS=m CONFIG_BINFMT_ELF=y CONFIG_COMPAT_BINFMT_ELF=y CONFIG_BINFMT_MISC=m -CONFIG_SPARC32_COMPAT=y CONFIG_COMPAT=y CONFIG_SYSVIPC_COMPAT=y CONFIG_SCHED_SMT=y @@ -744,13 +749,7 @@ CONFIG_DEVPORT=y CONFIG_I2C=y CONFIG_I2C_BOARDINFO=y # CONFIG_I2C_CHARDEV is not set - -# -# I2C Algorithms -# CONFIG_I2C_ALGOBIT=y -# CONFIG_I2C_ALGOPCF is not set -# CONFIG_I2C_ALGOPCA is not set # # I2C Hardware Bus support @@ -778,6 +777,7 @@ CONFIG_I2C_ALGOBIT=y # CONFIG_I2C_VIA is not set # CONFIG_I2C_VIAPRO is not set # CONFIG_I2C_VOODOO3 is not set +# CONFIG_I2C_PCA_PLATFORM is not set # # Miscellaneous I2C Chip support @@ -787,7 +787,6 @@ CONFIG_I2C_ALGOBIT=y # CONFIG_SENSORS_PCF8574 is not set # CONFIG_PCF8575 is not set # CONFIG_SENSORS_PCF8591 is not set -# CONFIG_TPS65010 is not set # CONFIG_SENSORS_MAX6875 is not set # CONFIG_SENSORS_TSL2550 is not set # CONFIG_I2C_DEBUG_CORE is not set @@ -869,6 +868,7 @@ CONFIG_SSB_POSSIBLE=y # Multifunction device drivers # # CONFIG_MFD_SM501 is not set +# CONFIG_HTC_PASIC3 is not set # # Multimedia devices @@ -1024,6 +1024,7 @@ CONFIG_SND_ALI5451=m # CONFIG_SND_AU8810 is not set # CONFIG_SND_AU8820 is not set # CONFIG_SND_AU8830 is not set +# CONFIG_SND_AW2 is not set # CONFIG_SND_AZT3328 is not set # CONFIG_SND_BT87X is not set # CONFIG_SND_CA0106 is not set @@ -1095,10 +1096,6 @@ CONFIG_SND_SUN_CS4231=m # CONFIG_SND_SOC is not set # -# SoC Audio support for SuperH -# - -# # ALSA SoC audio for Freescale SOCs # @@ -1219,10 +1216,6 @@ CONFIG_USB_STORAGE=m # CONFIG_NEW_LEDS is not set # CONFIG_INFINIBAND is not set # CONFIG_RTC_CLASS is not set - -# -# Userspace I/O -# # CONFIG_UIO is not set # @@ -1399,6 +1392,7 @@ CONFIG_SCHEDSTATS=y CONFIG_DEBUG_BUGVERBOSE=y # CONFIG_DEBUG_INFO is not set # CONFIG_DEBUG_VM is not set +# CONFIG_DEBUG_WRITECOUNT is not set # CONFIG_DEBUG_LIST is not set # CONFIG_DEBUG_SG is not set # CONFIG_BOOT_PRINTK_DELAY is not set @@ -1425,53 +1419,82 @@ CONFIG_ASYNC_CORE=m CONFIG_ASYNC_MEMCPY=m CONFIG_ASYNC_XOR=m CONFIG_CRYPTO=y + +# +# Crypto core or helper +# CONFIG_CRYPTO_ALGAPI=y CONFIG_CRYPTO_AEAD=y CONFIG_CRYPTO_BLKCIPHER=y -# CONFIG_CRYPTO_SEQIV is not set CONFIG_CRYPTO_HASH=y CONFIG_CRYPTO_MANAGER=y +CONFIG_CRYPTO_GF128MUL=m +CONFIG_CRYPTO_NULL=m +# CONFIG_CRYPTO_CRYPTD is not set +CONFIG_CRYPTO_AUTHENC=y +CONFIG_CRYPTO_TEST=m + +# +# Authenticated Encryption with Associated Data +# +# CONFIG_CRYPTO_CCM is not set +# CONFIG_CRYPTO_GCM is not set +# CONFIG_CRYPTO_SEQIV is not set + +# +# Block modes +# +CONFIG_CRYPTO_CBC=y +# CONFIG_CRYPTO_CTR is not set +# CONFIG_CRYPTO_CTS is not set +CONFIG_CRYPTO_ECB=m +CONFIG_CRYPTO_LRW=m +CONFIG_CRYPTO_PCBC=m +CONFIG_CRYPTO_XTS=m + +# +# Hash modes +# CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_XCBC=y -CONFIG_CRYPTO_NULL=m + +# +# Digest +# +CONFIG_CRYPTO_CRC32C=m CONFIG_CRYPTO_MD4=y CONFIG_CRYPTO_MD5=y +CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_SHA1=y CONFIG_CRYPTO_SHA256=m CONFIG_CRYPTO_SHA512=m -CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_TGR192=m -CONFIG_CRYPTO_GF128MUL=m -CONFIG_CRYPTO_ECB=m -CONFIG_CRYPTO_CBC=y -CONFIG_CRYPTO_PCBC=m -CONFIG_CRYPTO_LRW=m -CONFIG_CRYPTO_XTS=m -# CONFIG_CRYPTO_CTR is not set -# CONFIG_CRYPTO_GCM is not set -# CONFIG_CRYPTO_CCM is not set -# CONFIG_CRYPTO_CRYPTD is not set -CONFIG_CRYPTO_DES=y -CONFIG_CRYPTO_FCRYPT=m -CONFIG_CRYPTO_BLOWFISH=m -CONFIG_CRYPTO_TWOFISH=m -CONFIG_CRYPTO_TWOFISH_COMMON=m -CONFIG_CRYPTO_SERPENT=m +CONFIG_CRYPTO_WP512=m + +# +# Ciphers +# CONFIG_CRYPTO_AES=m +CONFIG_CRYPTO_ANUBIS=m +CONFIG_CRYPTO_ARC4=m +CONFIG_CRYPTO_BLOWFISH=m +CONFIG_CRYPTO_CAMELLIA=m CONFIG_CRYPTO_CAST5=m CONFIG_CRYPTO_CAST6=m -CONFIG_CRYPTO_TEA=m -CONFIG_CRYPTO_ARC4=m +CONFIG_CRYPTO_DES=y +CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_KHAZAD=m -CONFIG_CRYPTO_ANUBIS=m -CONFIG_CRYPTO_SEED=m # CONFIG_CRYPTO_SALSA20 is not set +CONFIG_CRYPTO_SEED=m +CONFIG_CRYPTO_SERPENT=m +CONFIG_CRYPTO_TEA=m +CONFIG_CRYPTO_TWOFISH=m +CONFIG_CRYPTO_TWOFISH_COMMON=m + +# +# Compression +# CONFIG_CRYPTO_DEFLATE=y -CONFIG_CRYPTO_MICHAEL_MIC=m -CONFIG_CRYPTO_CRC32C=m -CONFIG_CRYPTO_CAMELLIA=m -CONFIG_CRYPTO_TEST=m -CONFIG_CRYPTO_AUTHENC=y # CONFIG_CRYPTO_LZO is not set CONFIG_CRYPTO_HW=y # CONFIG_CRYPTO_DEV_HIFN_795X is not set @@ -1492,3 +1515,4 @@ CONFIG_PLIST=y CONFIG_HAS_IOMEM=y CONFIG_HAS_IOPORT=y CONFIG_HAS_DMA=y +CONFIG_HAVE_LMB=y diff --git a/arch/sparc64/kernel/Makefile b/arch/sparc64/kernel/Makefile index 63c6ae0dd27..ec4f5ebb1ca 100644 --- a/arch/sparc64/kernel/Makefile +++ b/arch/sparc64/kernel/Makefile @@ -15,17 +15,18 @@ obj-y := process.o setup.o cpu.o idprom.o \ visemul.o prom.o of_device.o hvapi.o sstate.o mdesc.o obj-$(CONFIG_STACKTRACE) += stacktrace.o -obj-$(CONFIG_PCI) += ebus.o isa.o pci_common.o \ +obj-$(CONFIG_PCI) += ebus.o pci_common.o \ pci_psycho.o pci_sabre.o pci_schizo.o \ pci_sun4v.o pci_sun4v_asm.o pci_fire.o obj-$(CONFIG_PCI_MSI) += pci_msi.o obj-$(CONFIG_SMP) += smp.o trampoline.o hvtramp.o -obj-$(CONFIG_SPARC32_COMPAT) += sys32.o sys_sparc32.o signal32.o +obj-$(CONFIG_COMPAT) += sys32.o sys_sparc32.o signal32.o obj-$(CONFIG_MODULES) += module.o obj-$(CONFIG_US3_FREQ) += us3_cpufreq.o obj-$(CONFIG_US2E_FREQ) += us2e_cpufreq.o obj-$(CONFIG_KPROBES) += kprobes.o obj-$(CONFIG_SUN_LDOMS) += ldc.o vio.o viohs.o ds.o obj-$(CONFIG_AUDIT) += audit.o -obj-$(CONFIG_AUDIT)$(CONFIG_SPARC32_COMPAT) += compat_audit.o +obj-$(CONFIG_AUDIT)$(CONFIG_COMPAT) += compat_audit.o obj-y += $(obj-yy) +obj-$(CONFIG_KGDB) += kgdb.o diff --git a/arch/sparc64/kernel/audit.c b/arch/sparc64/kernel/audit.c index 24d7f4b4178..8fff0ac63d5 100644 --- a/arch/sparc64/kernel/audit.c +++ b/arch/sparc64/kernel/audit.c @@ -30,7 +30,7 @@ static unsigned signal_class[] = { int audit_classify_arch(int arch) { -#ifdef CONFIG_SPARC32_COMPAT +#ifdef CONFIG_COMPAT if (arch == AUDIT_ARCH_SPARC) return 1; #endif @@ -39,7 +39,7 @@ int audit_classify_arch(int arch) int audit_classify_syscall(int abi, unsigned syscall) { -#ifdef CONFIG_SPARC32_COMPAT +#ifdef CONFIG_COMPAT extern int sparc32_classify_syscall(unsigned); if (abi == AUDIT_ARCH_SPARC) return sparc32_classify_syscall(syscall); @@ -60,7 +60,7 @@ int audit_classify_syscall(int abi, unsigned syscall) static int __init audit_classes_init(void) { -#ifdef CONFIG_SPARC32_COMPAT +#ifdef CONFIG_COMPAT extern __u32 sparc32_dir_class[]; extern __u32 sparc32_write_class[]; extern __u32 sparc32_read_class[]; diff --git a/arch/sparc64/kernel/cherrs.S b/arch/sparc64/kernel/cherrs.S new file mode 100644 index 00000000000..89afebd7eca --- /dev/null +++ b/arch/sparc64/kernel/cherrs.S @@ -0,0 +1,579 @@ + /* These get patched into the trap table at boot time + * once we know we have a cheetah processor. + */ + .globl cheetah_fecc_trap_vector + .type cheetah_fecc_trap_vector,#function +cheetah_fecc_trap_vector: + membar #Sync + ldxa [%g0] ASI_DCU_CONTROL_REG, %g1 + andn %g1, DCU_DC | DCU_IC, %g1 + stxa %g1, [%g0] ASI_DCU_CONTROL_REG + membar #Sync + sethi %hi(cheetah_fast_ecc), %g2 + jmpl %g2 + %lo(cheetah_fast_ecc), %g0 + mov 0, %g1 + .size cheetah_fecc_trap_vector,.-cheetah_fecc_trap_vector + + .globl cheetah_fecc_trap_vector_tl1 + .type cheetah_fecc_trap_vector_tl1,#function +cheetah_fecc_trap_vector_tl1: + membar #Sync + ldxa [%g0] ASI_DCU_CONTROL_REG, %g1 + andn %g1, DCU_DC | DCU_IC, %g1 + stxa %g1, [%g0] ASI_DCU_CONTROL_REG + membar #Sync + sethi %hi(cheetah_fast_ecc), %g2 + jmpl %g2 + %lo(cheetah_fast_ecc), %g0 + mov 1, %g1 + .size cheetah_fecc_trap_vector_tl1,.-cheetah_fecc_trap_vector_tl1 + + .globl cheetah_cee_trap_vector + .type cheetah_cee_trap_vector,#function +cheetah_cee_trap_vector: + membar #Sync + ldxa [%g0] ASI_DCU_CONTROL_REG, %g1 + andn %g1, DCU_IC, %g1 + stxa %g1, [%g0] ASI_DCU_CONTROL_REG + membar #Sync + sethi %hi(cheetah_cee), %g2 + jmpl %g2 + %lo(cheetah_cee), %g0 + mov 0, %g1 + .size cheetah_cee_trap_vector,.-cheetah_cee_trap_vector + + .globl cheetah_cee_trap_vector_tl1 + .type cheetah_cee_trap_vector_tl1,#function +cheetah_cee_trap_vector_tl1: + membar #Sync + ldxa [%g0] ASI_DCU_CONTROL_REG, %g1 + andn %g1, DCU_IC, %g1 + stxa %g1, [%g0] ASI_DCU_CONTROL_REG + membar #Sync + sethi %hi(cheetah_cee), %g2 + jmpl %g2 + %lo(cheetah_cee), %g0 + mov 1, %g1 + .size cheetah_cee_trap_vector_tl1,.-cheetah_cee_trap_vector_tl1 + + .globl cheetah_deferred_trap_vector + .type cheetah_deferred_trap_vector,#function +cheetah_deferred_trap_vector: + membar #Sync + ldxa [%g0] ASI_DCU_CONTROL_REG, %g1; + andn %g1, DCU_DC | DCU_IC, %g1; + stxa %g1, [%g0] ASI_DCU_CONTROL_REG; + membar #Sync; + sethi %hi(cheetah_deferred_trap), %g2 + jmpl %g2 + %lo(cheetah_deferred_trap), %g0 + mov 0, %g1 + .size cheetah_deferred_trap_vector,.-cheetah_deferred_trap_vector + + .globl cheetah_deferred_trap_vector_tl1 + .type cheetah_deferred_trap_vector_tl1,#function +cheetah_deferred_trap_vector_tl1: + membar #Sync; + ldxa [%g0] ASI_DCU_CONTROL_REG, %g1; + andn %g1, DCU_DC | DCU_IC, %g1; + stxa %g1, [%g0] ASI_DCU_CONTROL_REG; + membar #Sync; + sethi %hi(cheetah_deferred_trap), %g2 + jmpl %g2 + %lo(cheetah_deferred_trap), %g0 + mov 1, %g1 + .size cheetah_deferred_trap_vector_tl1,.-cheetah_deferred_trap_vector_tl1 + + /* Cheetah+ specific traps. These are for the new I/D cache parity + * error traps. The first argument to cheetah_plus_parity_handler + * is encoded as follows: + * + * Bit0: 0=dcache,1=icache + * Bit1: 0=recoverable,1=unrecoverable + */ + .globl cheetah_plus_dcpe_trap_vector + .type cheetah_plus_dcpe_trap_vector,#function +cheetah_plus_dcpe_trap_vector: + membar #Sync + sethi %hi(do_cheetah_plus_data_parity), %g7 + jmpl %g7 + %lo(do_cheetah_plus_data_parity), %g0 + nop + nop + nop + nop + nop + .size cheetah_plus_dcpe_trap_vector,.-cheetah_plus_dcpe_trap_vector + + .type do_cheetah_plus_data_parity,#function +do_cheetah_plus_data_parity: + rdpr %pil, %g2 + wrpr %g0, 15, %pil + ba,pt %xcc, etrap_irq + rd %pc, %g7 +#ifdef CONFIG_TRACE_IRQFLAGS + call trace_hardirqs_off + nop +#endif + mov 0x0, %o0 + call cheetah_plus_parity_error + add %sp, PTREGS_OFF, %o1 + ba,a,pt %xcc, rtrap_irq + .size do_cheetah_plus_data_parity,.-do_cheetah_plus_data_parity + + .globl cheetah_plus_dcpe_trap_vector_tl1 + .type cheetah_plus_dcpe_trap_vector_tl1,#function +cheetah_plus_dcpe_trap_vector_tl1: + membar #Sync + wrpr PSTATE_IG | PSTATE_PEF | PSTATE_PRIV, %pstate + sethi %hi(do_dcpe_tl1), %g3 + jmpl %g3 + %lo(do_dcpe_tl1), %g0 + nop + nop + nop + nop + .size cheetah_plus_dcpe_trap_vector_tl1,.-cheetah_plus_dcpe_trap_vector_tl1 + + .globl cheetah_plus_icpe_trap_vector + .type cheetah_plus_icpe_trap_vector,#function +cheetah_plus_icpe_trap_vector: + membar #Sync + sethi %hi(do_cheetah_plus_insn_parity), %g7 + jmpl %g7 + %lo(do_cheetah_plus_insn_parity), %g0 + nop + nop + nop + nop + nop + .size cheetah_plus_icpe_trap_vector,.-cheetah_plus_icpe_trap_vector + + .type do_cheetah_plus_insn_parity,#function +do_cheetah_plus_insn_parity: + rdpr %pil, %g2 + wrpr %g0, 15, %pil + ba,pt %xcc, etrap_irq + rd %pc, %g7 +#ifdef CONFIG_TRACE_IRQFLAGS + call trace_hardirqs_off + nop +#endif + mov 0x1, %o0 + call cheetah_plus_parity_error + add %sp, PTREGS_OFF, %o1 + ba,a,pt %xcc, rtrap_irq + .size do_cheetah_plus_insn_parity,.-do_cheetah_plus_insn_parity + + .globl cheetah_plus_icpe_trap_vector_tl1 + .type cheetah_plus_icpe_trap_vector_tl1,#function +cheetah_plus_icpe_trap_vector_tl1: + membar #Sync + wrpr PSTATE_IG | PSTATE_PEF | PSTATE_PRIV, %pstate + sethi %hi(do_icpe_tl1), %g3 + jmpl %g3 + %lo(do_icpe_tl1), %g0 + nop + nop + nop + nop + .size cheetah_plus_icpe_trap_vector_tl1,.-cheetah_plus_icpe_trap_vector_tl1 + + /* If we take one of these traps when tl >= 1, then we + * jump to interrupt globals. If some trap level above us + * was also using interrupt globals, we cannot recover. + * We may use all interrupt global registers except %g6. + */ + .globl do_dcpe_tl1 + .type do_dcpe_tl1,#function +do_dcpe_tl1: + rdpr %tl, %g1 ! Save original trap level + mov 1, %g2 ! Setup TSTATE checking loop + sethi %hi(TSTATE_IG), %g3 ! TSTATE mask bit +1: wrpr %g2, %tl ! Set trap level to check + rdpr %tstate, %g4 ! Read TSTATE for this level + andcc %g4, %g3, %g0 ! Interrupt globals in use? + bne,a,pn %xcc, do_dcpe_tl1_fatal ! Yep, irrecoverable + wrpr %g1, %tl ! Restore original trap level + add %g2, 1, %g2 ! Next trap level + cmp %g2, %g1 ! Hit them all yet? + ble,pt %icc, 1b ! Not yet + nop + wrpr %g1, %tl ! Restore original trap level +do_dcpe_tl1_nonfatal: /* Ok we may use interrupt globals safely. */ + sethi %hi(dcache_parity_tl1_occurred), %g2 + lduw [%g2 + %lo(dcache_parity_tl1_occurred)], %g1 + add %g1, 1, %g1 + stw %g1, [%g2 + %lo(dcache_parity_tl1_occurred)] + /* Reset D-cache parity */ + sethi %hi(1 << 16), %g1 ! D-cache size + mov (1 << 5), %g2 ! D-cache line size + sub %g1, %g2, %g1 ! Move down 1 cacheline +1: srl %g1, 14, %g3 ! Compute UTAG + membar #Sync + stxa %g3, [%g1] ASI_DCACHE_UTAG + membar #Sync + sub %g2, 8, %g3 ! 64-bit data word within line +2: membar #Sync + stxa %g0, [%g1 + %g3] ASI_DCACHE_DATA + membar #Sync + subcc %g3, 8, %g3 ! Next 64-bit data word + bge,pt %icc, 2b + nop + subcc %g1, %g2, %g1 ! Next cacheline + bge,pt %icc, 1b + nop + ba,pt %xcc, dcpe_icpe_tl1_common + nop + +do_dcpe_tl1_fatal: + sethi %hi(1f), %g7 + ba,pt %xcc, etraptl1 +1: or %g7, %lo(1b), %g7 + mov 0x2, %o0 + call cheetah_plus_parity_error + add %sp, PTREGS_OFF, %o1 + ba,pt %xcc, rtrap + nop + .size do_dcpe_tl1,.-do_dcpe_tl1 + + .globl do_icpe_tl1 + .type do_icpe_tl1,#function +do_icpe_tl1: + rdpr %tl, %g1 ! Save original trap level + mov 1, %g2 ! Setup TSTATE checking loop + sethi %hi(TSTATE_IG), %g3 ! TSTATE mask bit +1: wrpr %g2, %tl ! Set trap level to check + rdpr %tstate, %g4 ! Read TSTATE for this level + andcc %g4, %g3, %g0 ! Interrupt globals in use? + bne,a,pn %xcc, do_icpe_tl1_fatal ! Yep, irrecoverable + wrpr %g1, %tl ! Restore original trap level + add %g2, 1, %g2 ! Next trap level + cmp %g2, %g1 ! Hit them all yet? + ble,pt %icc, 1b ! Not yet + nop + wrpr %g1, %tl ! Restore original trap level +do_icpe_tl1_nonfatal: /* Ok we may use interrupt globals safely. */ + sethi %hi(icache_parity_tl1_occurred), %g2 + lduw [%g2 + %lo(icache_parity_tl1_occurred)], %g1 + add %g1, 1, %g1 + stw %g1, [%g2 + %lo(icache_parity_tl1_occurred)] + /* Flush I-cache */ + sethi %hi(1 << 15), %g1 ! I-cache size + mov (1 << 5), %g2 ! I-cache line size + sub %g1, %g2, %g1 +1: or %g1, (2 << 3), %g3 + stxa %g0, [%g3] ASI_IC_TAG + membar #Sync + subcc %g1, %g2, %g1 + bge,pt %icc, 1b + nop + ba,pt %xcc, dcpe_icpe_tl1_common + nop + +do_icpe_tl1_fatal: + sethi %hi(1f), %g7 + ba,pt %xcc, etraptl1 +1: or %g7, %lo(1b), %g7 + mov 0x3, %o0 + call cheetah_plus_parity_error + add %sp, PTREGS_OFF, %o1 + ba,pt %xcc, rtrap + nop + .size do_icpe_tl1,.-do_icpe_tl1 + + .type dcpe_icpe_tl1_common,#function +dcpe_icpe_tl1_common: + /* Flush D-cache, re-enable D/I caches in DCU and finally + * retry the trapping instruction. + */ + sethi %hi(1 << 16), %g1 ! D-cache size + mov (1 << 5), %g2 ! D-cache line size + sub %g1, %g2, %g1 +1: stxa %g0, [%g1] ASI_DCACHE_TAG + membar #Sync + subcc %g1, %g2, %g1 + bge,pt %icc, 1b + nop + ldxa [%g0] ASI_DCU_CONTROL_REG, %g1 + or %g1, (DCU_DC | DCU_IC), %g1 + stxa %g1, [%g0] ASI_DCU_CONTROL_REG + membar #Sync + retry + .size dcpe_icpe_tl1_common,.-dcpe_icpe_tl1_common + + /* Capture I/D/E-cache state into per-cpu error scoreboard. + * + * %g1: (TL>=0) ? 1 : 0 + * %g2: scratch + * %g3: scratch + * %g4: AFSR + * %g5: AFAR + * %g6: unused, will have current thread ptr after etrap + * %g7: scratch + */ + .type __cheetah_log_error,#function +__cheetah_log_error: + /* Put "TL1" software bit into AFSR. */ + and %g1, 0x1, %g1 + sllx %g1, 63, %g2 + or %g4, %g2, %g4 + + /* Get log entry pointer for this cpu at this trap level. */ + BRANCH_IF_JALAPENO(g2,g3,50f) + ldxa [%g0] ASI_SAFARI_CONFIG, %g2 + srlx %g2, 17, %g2 + ba,pt %xcc, 60f + and %g2, 0x3ff, %g2 + +50: ldxa [%g0] ASI_JBUS_CONFIG, %g2 + srlx %g2, 17, %g2 + and %g2, 0x1f, %g2 + +60: sllx %g2, 9, %g2 + sethi %hi(cheetah_error_log), %g3 + ldx [%g3 + %lo(cheetah_error_log)], %g3 + brz,pn %g3, 80f + nop + + add %g3, %g2, %g3 + sllx %g1, 8, %g1 + add %g3, %g1, %g1 + + /* %g1 holds pointer to the top of the logging scoreboard */ + ldx [%g1 + 0x0], %g7 + cmp %g7, -1 + bne,pn %xcc, 80f + nop + + stx %g4, [%g1 + 0x0] + stx %g5, [%g1 + 0x8] + add %g1, 0x10, %g1 + + /* %g1 now points to D-cache logging area */ + set 0x3ff8, %g2 /* DC_addr mask */ + and %g5, %g2, %g2 /* DC_addr bits of AFAR */ + srlx %g5, 12, %g3 + or %g3, 1, %g3 /* PHYS tag + valid */ + +10: ldxa [%g2] ASI_DCACHE_TAG, %g7 + cmp %g3, %g7 /* TAG match? */ + bne,pt %xcc, 13f + nop + + /* Yep, what we want, capture state. */ + stx %g2, [%g1 + 0x20] + stx %g7, [%g1 + 0x28] + + /* A membar Sync is required before and after utag access. */ + membar #Sync + ldxa [%g2] ASI_DCACHE_UTAG, %g7 + membar #Sync + stx %g7, [%g1 + 0x30] + ldxa [%g2] ASI_DCACHE_SNOOP_TAG, %g7 + stx %g7, [%g1 + 0x38] + clr %g3 + +12: ldxa [%g2 + %g3] ASI_DCACHE_DATA, %g7 + stx %g7, [%g1] + add %g3, (1 << 5), %g3 + cmp %g3, (4 << 5) + bl,pt %xcc, 12b + add %g1, 0x8, %g1 + + ba,pt %xcc, 20f + add %g1, 0x20, %g1 + +13: sethi %hi(1 << 14), %g7 + add %g2, %g7, %g2 + srlx %g2, 14, %g7 + cmp %g7, 4 + bl,pt %xcc, 10b + nop + + add %g1, 0x40, %g1 + + /* %g1 now points to I-cache logging area */ +20: set 0x1fe0, %g2 /* IC_addr mask */ + and %g5, %g2, %g2 /* IC_addr bits of AFAR */ + sllx %g2, 1, %g2 /* IC_addr[13:6]==VA[12:5] */ + srlx %g5, (13 - 8), %g3 /* Make PTAG */ + andn %g3, 0xff, %g3 /* Mask off undefined bits */ + +21: ldxa [%g2] ASI_IC_TAG, %g7 + andn %g7, 0xff, %g7 + cmp %g3, %g7 + bne,pt %xcc, 23f + nop + + /* Yep, what we want, capture state. */ + stx %g2, [%g1 + 0x40] + stx %g7, [%g1 + 0x48] + add %g2, (1 << 3), %g2 + ldxa [%g2] ASI_IC_TAG, %g7 + add %g2, (1 << 3), %g2 + stx %g7, [%g1 + 0x50] + ldxa [%g2] ASI_IC_TAG, %g7 + add %g2, (1 << 3), %g2 + stx %g7, [%g1 + 0x60] + ldxa [%g2] ASI_IC_TAG, %g7 + stx %g7, [%g1 + 0x68] + sub %g2, (3 << 3), %g2 + ldxa [%g2] ASI_IC_STAG, %g7 + stx %g7, [%g1 + 0x58] + clr %g3 + srlx %g2, 2, %g2 + +22: ldxa [%g2 + %g3] ASI_IC_INSTR, %g7 + stx %g7, [%g1] + add %g3, (1 << 3), %g3 + cmp %g3, (8 << 3) + bl,pt %xcc, 22b + add %g1, 0x8, %g1 + + ba,pt %xcc, 30f + add %g1, 0x30, %g1 + +23: sethi %hi(1 << 14), %g7 + add %g2, %g7, %g2 + srlx %g2, 14, %g7 + cmp %g7, 4 + bl,pt %xcc, 21b + nop + + add %g1, 0x70, %g1 + + /* %g1 now points to E-cache logging area */ +30: andn %g5, (32 - 1), %g2 + stx %g2, [%g1 + 0x20] + ldxa [%g2] ASI_EC_TAG_DATA, %g7 + stx %g7, [%g1 + 0x28] + ldxa [%g2] ASI_EC_R, %g0 + clr %g3 + +31: ldxa [%g3] ASI_EC_DATA, %g7 + stx %g7, [%g1 + %g3] + add %g3, 0x8, %g3 + cmp %g3, 0x20 + + bl,pt %xcc, 31b + nop +80: + rdpr %tt, %g2 + cmp %g2, 0x70 + be c_fast_ecc + cmp %g2, 0x63 + be c_cee + nop + ba,pt %xcc, c_deferred + .size __cheetah_log_error,.-__cheetah_log_error + + /* Cheetah FECC trap handling, we get here from tl{0,1}_fecc + * in the trap table. That code has done a memory barrier + * and has disabled both the I-cache and D-cache in the DCU + * control register. The I-cache is disabled so that we may + * capture the corrupted cache line, and the D-cache is disabled + * because corrupt data may have been placed there and we don't + * want to reference it. + * + * %g1 is one if this trap occurred at %tl >= 1. + * + * Next, we turn off error reporting so that we don't recurse. + */ + .globl cheetah_fast_ecc + .type cheetah_fast_ecc,#function +cheetah_fast_ecc: + ldxa [%g0] ASI_ESTATE_ERROR_EN, %g2 + andn %g2, ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN, %g2 + stxa %g2, [%g0] ASI_ESTATE_ERROR_EN + membar #Sync + + /* Fetch and clear AFSR/AFAR */ + ldxa [%g0] ASI_AFSR, %g4 + ldxa [%g0] ASI_AFAR, %g5 + stxa %g4, [%g0] ASI_AFSR + membar #Sync + + ba,pt %xcc, __cheetah_log_error + nop + .size cheetah_fast_ecc,.-cheetah_fast_ecc + + .type c_fast_ecc,#function +c_fast_ecc: + rdpr %pil, %g2 + wrpr %g0, 15, %pil + ba,pt %xcc, etrap_irq + rd %pc, %g7 +#ifdef CONFIG_TRACE_IRQFLAGS + call trace_hardirqs_off + nop +#endif + mov %l4, %o1 + mov %l5, %o2 + call cheetah_fecc_handler + add %sp, PTREGS_OFF, %o0 + ba,a,pt %xcc, rtrap_irq + .size c_fast_ecc,.-c_fast_ecc + + /* Our caller has disabled I-cache and performed membar Sync. */ + .globl cheetah_cee + .type cheetah_cee,#function +cheetah_cee: + ldxa [%g0] ASI_ESTATE_ERROR_EN, %g2 + andn %g2, ESTATE_ERROR_CEEN, %g2 + stxa %g2, [%g0] ASI_ESTATE_ERROR_EN + membar #Sync + + /* Fetch and clear AFSR/AFAR */ + ldxa [%g0] ASI_AFSR, %g4 + ldxa [%g0] ASI_AFAR, %g5 + stxa %g4, [%g0] ASI_AFSR + membar #Sync + + ba,pt %xcc, __cheetah_log_error + nop + .size cheetah_cee,.-cheetah_cee + + .type c_cee,#function +c_cee: + rdpr %pil, %g2 + wrpr %g0, 15, %pil + ba,pt %xcc, etrap_irq + rd %pc, %g7 +#ifdef CONFIG_TRACE_IRQFLAGS + call trace_hardirqs_off + nop +#endif + mov %l4, %o1 + mov %l5, %o2 + call cheetah_cee_handler + add %sp, PTREGS_OFF, %o0 + ba,a,pt %xcc, rtrap_irq + .size c_cee,.-c_cee + + /* Our caller has disabled I-cache+D-cache and performed membar Sync. */ + .globl cheetah_deferred_trap + .type cheetah_deferred_trap,#function +cheetah_deferred_trap: + ldxa [%g0] ASI_ESTATE_ERROR_EN, %g2 + andn %g2, ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN, %g2 + stxa %g2, [%g0] ASI_ESTATE_ERROR_EN + membar #Sync + + /* Fetch and clear AFSR/AFAR */ + ldxa [%g0] ASI_AFSR, %g4 + ldxa [%g0] ASI_AFAR, %g5 + stxa %g4, [%g0] ASI_AFSR + membar #Sync + + ba,pt %xcc, __cheetah_log_error + nop + .size cheetah_deferred_trap,.-cheetah_deferred_trap + + .type c_deferred,#function +c_deferred: + rdpr %pil, %g2 + wrpr %g0, 15, %pil + ba,pt %xcc, etrap_irq + rd %pc, %g7 +#ifdef CONFIG_TRACE_IRQFLAGS + call trace_hardirqs_off + nop +#endif + mov %l4, %o1 + mov %l5, %o2 + call cheetah_deferred_handler + add %sp, PTREGS_OFF, %o0 + ba,a,pt %xcc, rtrap_irq + .size c_deferred,.-c_deferred diff --git a/arch/sparc64/kernel/ebus.c b/arch/sparc64/kernel/ebus.c index 04ab81cb4f4..bc263227484 100644 --- a/arch/sparc64/kernel/ebus.c +++ b/arch/sparc64/kernel/ebus.c @@ -396,6 +396,7 @@ static void __init fill_ebus_device(struct device_node *dp, struct linux_ebus_de sd->op = &dev->ofdev; sd->iommu = dev->bus->ofdev.dev.parent->archdata.iommu; sd->stc = dev->bus->ofdev.dev.parent->archdata.stc; + sd->numa_node = dev->bus->ofdev.dev.parent->archdata.numa_node; dev->ofdev.node = dp; dev->ofdev.dev.parent = &dev->bus->ofdev.dev; diff --git a/arch/sparc64/kernel/entry.S b/arch/sparc64/kernel/entry.S deleted file mode 100644 index fb43c76bdc2..00000000000 --- a/arch/sparc64/kernel/entry.S +++ /dev/null @@ -1,2576 +0,0 @@ -/* $Id: entry.S,v 1.144 2002/02/09 19:49:30 davem Exp $ - * arch/sparc64/kernel/entry.S: Sparc64 trap low-level entry points. - * - * Copyright (C) 1995,1997 David S. Miller (davem@caip.rutgers.edu) - * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be) - * Copyright (C) 1996 Miguel de Icaza (miguel@nuclecu.unam.mx) - * Copyright (C) 1996,98,99 Jakub Jelinek (jj@sunsite.mff.cuni.cz) - */ - -#include <linux/errno.h> - -#include <asm/head.h> -#include <asm/asi.h> -#include <asm/smp.h> -#include <asm/ptrace.h> -#include <asm/page.h> -#include <asm/signal.h> -#include <asm/pgtable.h> -#include <asm/processor.h> -#include <asm/visasm.h> -#include <asm/estate.h> -#include <asm/auxio.h> -#include <asm/sfafsr.h> -#include <asm/pil.h> -#include <asm/unistd.h> - -#define curptr g6 - - .text - .align 32 - - /* This is trivial with the new code... */ - .globl do_fpdis -do_fpdis: - sethi %hi(TSTATE_PEF), %g4 - rdpr %tstate, %g5 - andcc %g5, %g4, %g0 - be,pt %xcc, 1f - nop - rd %fprs, %g5 - andcc %g5, FPRS_FEF, %g0 - be,pt %xcc, 1f - nop - - /* Legal state when DCR_IFPOE is set in Cheetah %dcr. */ - sethi %hi(109f), %g7 - ba,pt %xcc, etrap -109: or %g7, %lo(109b), %g7 - add %g0, %g0, %g0 - ba,a,pt %xcc, rtrap_clr_l6 - -1: TRAP_LOAD_THREAD_REG(%g6, %g1) - ldub [%g6 + TI_FPSAVED], %g5 - wr %g0, FPRS_FEF, %fprs - andcc %g5, FPRS_FEF, %g0 - be,a,pt %icc, 1f - clr %g7 - ldx [%g6 + TI_GSR], %g7 -1: andcc %g5, FPRS_DL, %g0 - bne,pn %icc, 2f - fzero %f0 - andcc %g5, FPRS_DU, %g0 - bne,pn %icc, 1f - fzero %f2 - faddd %f0, %f2, %f4 - fmuld %f0, %f2, %f6 - faddd %f0, %f2, %f8 - fmuld %f0, %f2, %f10 - faddd %f0, %f2, %f12 - fmuld %f0, %f2, %f14 - faddd %f0, %f2, %f16 - fmuld %f0, %f2, %f18 - faddd %f0, %f2, %f20 - fmuld %f0, %f2, %f22 - faddd %f0, %f2, %f24 - fmuld %f0, %f2, %f26 - faddd %f0, %f2, %f28 - fmuld %f0, %f2, %f30 - faddd %f0, %f2, %f32 - fmuld %f0, %f2, %f34 - faddd %f0, %f2, %f36 - fmuld %f0, %f2, %f38 - faddd %f0, %f2, %f40 - fmuld %f0, %f2, %f42 - faddd %f0, %f2, %f44 - fmuld %f0, %f2, %f46 - faddd %f0, %f2, %f48 - fmuld %f0, %f2, %f50 - faddd %f0, %f2, %f52 - fmuld %f0, %f2, %f54 - faddd %f0, %f2, %f56 - fmuld %f0, %f2, %f58 - b,pt %xcc, fpdis_exit2 - faddd %f0, %f2, %f60 -1: mov SECONDARY_CONTEXT, %g3 - add %g6, TI_FPREGS + 0x80, %g1 - faddd %f0, %f2, %f4 - fmuld %f0, %f2, %f6 - -661: ldxa [%g3] ASI_DMMU, %g5 - .section .sun4v_1insn_patch, "ax" - .word 661b - ldxa [%g3] ASI_MMU, %g5 - .previous - - sethi %hi(sparc64_kern_sec_context), %g2 - ldx [%g2 + %lo(sparc64_kern_sec_context)], %g2 - -661: stxa %g2, [%g3] ASI_DMMU - .section .sun4v_1insn_patch, "ax" - .word 661b - stxa %g2, [%g3] ASI_MMU - .previous - - membar #Sync - add %g6, TI_FPREGS + 0xc0, %g2 - faddd %f0, %f2, %f8 - fmuld %f0, %f2, %f10 - membar #Sync - ldda [%g1] ASI_BLK_S, %f32 - ldda [%g2] ASI_BLK_S, %f48 - membar #Sync - faddd %f0, %f2, %f12 - fmuld %f0, %f2, %f14 - faddd %f0, %f2, %f16 - fmuld %f0, %f2, %f18 - faddd %f0, %f2, %f20 - fmuld %f0, %f2, %f22 - faddd %f0, %f2, %f24 - fmuld %f0, %f2, %f26 - faddd %f0, %f2, %f28 - fmuld %f0, %f2, %f30 - b,pt %xcc, fpdis_exit - nop -2: andcc %g5, FPRS_DU, %g0 - bne,pt %icc, 3f - fzero %f32 - mov SECONDARY_CONTEXT, %g3 - fzero %f34 - -661: ldxa [%g3] ASI_DMMU, %g5 - .section .sun4v_1insn_patch, "ax" - .word 661b - ldxa [%g3] ASI_MMU, %g5 - .previous - - add %g6, TI_FPREGS, %g1 - sethi %hi(sparc64_kern_sec_context), %g2 - ldx [%g2 + %lo(sparc64_kern_sec_context)], %g2 - -661: stxa %g2, [%g3] ASI_DMMU - .section .sun4v_1insn_patch, "ax" - .word 661b - stxa %g2, [%g3] ASI_MMU - .previous - - membar #Sync - add %g6, TI_FPREGS + 0x40, %g2 - faddd %f32, %f34, %f36 - fmuld %f32, %f34, %f38 - membar #Sync - ldda [%g1] ASI_BLK_S, %f0 - ldda [%g2] ASI_BLK_S, %f16 - membar #Sync - faddd %f32, %f34, %f40 - fmuld %f32, %f34, %f42 - faddd %f32, %f34, %f44 - fmuld %f32, %f34, %f46 - faddd %f32, %f34, %f48 - fmuld %f32, %f34, %f50 - faddd %f32, %f34, %f52 - fmuld %f32, %f34, %f54 - faddd %f32, %f34, %f56 - fmuld %f32, %f34, %f58 - faddd %f32, %f34, %f60 - fmuld %f32, %f34, %f62 - ba,pt %xcc, fpdis_exit - nop -3: mov SECONDARY_CONTEXT, %g3 - add %g6, TI_FPREGS, %g1 - -661: ldxa [%g3] ASI_DMMU, %g5 - .section .sun4v_1insn_patch, "ax" - .word 661b - ldxa [%g3] ASI_MMU, %g5 - .previous - - sethi %hi(sparc64_kern_sec_context), %g2 - ldx [%g2 + %lo(sparc64_kern_sec_context)], %g2 - -661: stxa %g2, [%g3] ASI_DMMU - .section .sun4v_1insn_patch, "ax" - .word 661b - stxa %g2, [%g3] ASI_MMU - .previous - - membar #Sync - mov 0x40, %g2 - membar #Sync - ldda [%g1] ASI_BLK_S, %f0 - ldda [%g1 + %g2] ASI_BLK_S, %f16 - add %g1, 0x80, %g1 - ldda [%g1] ASI_BLK_S, %f32 - ldda [%g1 + %g2] ASI_BLK_S, %f48 - membar #Sync -fpdis_exit: - -661: stxa %g5, [%g3] ASI_DMMU - .section .sun4v_1insn_patch, "ax" - .word 661b - stxa %g5, [%g3] ASI_MMU - .previous - - membar #Sync -fpdis_exit2: - wr %g7, 0, %gsr - ldx [%g6 + TI_XFSR], %fsr - rdpr %tstate, %g3 - or %g3, %g4, %g3 ! anal... - wrpr %g3, %tstate - wr %g0, FPRS_FEF, %fprs ! clean DU/DL bits - retry - - .align 32 -fp_other_bounce: - call do_fpother - add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - clr %l6 - - .globl do_fpother_check_fitos - .align 32 -do_fpother_check_fitos: - TRAP_LOAD_THREAD_REG(%g6, %g1) - sethi %hi(fp_other_bounce - 4), %g7 - or %g7, %lo(fp_other_bounce - 4), %g7 - - /* NOTE: Need to preserve %g7 until we fully commit - * to the fitos fixup. - */ - stx %fsr, [%g6 + TI_XFSR] - rdpr %tstate, %g3 - andcc %g3, TSTATE_PRIV, %g0 - bne,pn %xcc, do_fptrap_after_fsr - nop - ldx [%g6 + TI_XFSR], %g3 - srlx %g3, 14, %g1 - and %g1, 7, %g1 - cmp %g1, 2 ! Unfinished FP-OP - bne,pn %xcc, do_fptrap_after_fsr - sethi %hi(1 << 23), %g1 ! Inexact - andcc %g3, %g1, %g0 - bne,pn %xcc, do_fptrap_after_fsr - rdpr %tpc, %g1 - lduwa [%g1] ASI_AIUP, %g3 ! This cannot ever fail -#define FITOS_MASK 0xc1f83fe0 -#define FITOS_COMPARE 0x81a01880 - sethi %hi(FITOS_MASK), %g1 - or %g1, %lo(FITOS_MASK), %g1 - and %g3, %g1, %g1 - sethi %hi(FITOS_COMPARE), %g2 - or %g2, %lo(FITOS_COMPARE), %g2 - cmp %g1, %g2 - bne,pn %xcc, do_fptrap_after_fsr - nop - std %f62, [%g6 + TI_FPREGS + (62 * 4)] - sethi %hi(fitos_table_1), %g1 - and %g3, 0x1f, %g2 - or %g1, %lo(fitos_table_1), %g1 - sllx %g2, 2, %g2 - jmpl %g1 + %g2, %g0 - ba,pt %xcc, fitos_emul_continue - -fitos_table_1: - fitod %f0, %f62 - fitod %f1, %f62 - fitod %f2, %f62 - fitod %f3, %f62 - fitod %f4, %f62 - fitod %f5, %f62 - fitod %f6, %f62 - fitod %f7, %f62 - fitod %f8, %f62 - fitod %f9, %f62 - fitod %f10, %f62 - fitod %f11, %f62 - fitod %f12, %f62 - fitod %f13, %f62 - fitod %f14, %f62 - fitod %f15, %f62 - fitod %f16, %f62 - fitod %f17, %f62 - fitod %f18, %f62 - fitod %f19, %f62 - fitod %f20, %f62 - fitod %f21, %f62 - fitod %f22, %f62 - fitod %f23, %f62 - fitod %f24, %f62 - fitod %f25, %f62 - fitod %f26, %f62 - fitod %f27, %f62 - fitod %f28, %f62 - fitod %f29, %f62 - fitod %f30, %f62 - fitod %f31, %f62 - -fitos_emul_continue: - sethi %hi(fitos_table_2), %g1 - srl %g3, 25, %g2 - or %g1, %lo(fitos_table_2), %g1 - and %g2, 0x1f, %g2 - sllx %g2, 2, %g2 - jmpl %g1 + %g2, %g0 - ba,pt %xcc, fitos_emul_fini - -fitos_table_2: - fdtos %f62, %f0 - fdtos %f62, %f1 - fdtos %f62, %f2 - fdtos %f62, %f3 - fdtos %f62, %f4 - fdtos %f62, %f5 - fdtos %f62, %f6 - fdtos %f62, %f7 - fdtos %f62, %f8 - fdtos %f62, %f9 - fdtos %f62, %f10 - fdtos %f62, %f11 - fdtos %f62, %f12 - fdtos %f62, %f13 - fdtos %f62, %f14 - fdtos %f62, %f15 - fdtos %f62, %f16 - fdtos %f62, %f17 - fdtos %f62, %f18 - fdtos %f62, %f19 - fdtos %f62, %f20 - fdtos %f62, %f21 - fdtos %f62, %f22 - fdtos %f62, %f23 - fdtos %f62, %f24 - fdtos %f62, %f25 - fdtos %f62, %f26 - fdtos %f62, %f27 - fdtos %f62, %f28 - fdtos %f62, %f29 - fdtos %f62, %f30 - fdtos %f62, %f31 - -fitos_emul_fini: - ldd [%g6 + TI_FPREGS + (62 * 4)], %f62 - done - - .globl do_fptrap - .align 32 -do_fptrap: - TRAP_LOAD_THREAD_REG(%g6, %g1) - stx %fsr, [%g6 + TI_XFSR] -do_fptrap_after_fsr: - ldub [%g6 + TI_FPSAVED], %g3 - rd %fprs, %g1 - or %g3, %g1, %g3 - stb %g3, [%g6 + TI_FPSAVED] - rd %gsr, %g3 - stx %g3, [%g6 + TI_GSR] - mov SECONDARY_CONTEXT, %g3 - -661: ldxa [%g3] ASI_DMMU, %g5 - .section .sun4v_1insn_patch, "ax" - .word 661b - ldxa [%g3] ASI_MMU, %g5 - .previous - - sethi %hi(sparc64_kern_sec_context), %g2 - ldx [%g2 + %lo(sparc64_kern_sec_context)], %g2 - -661: stxa %g2, [%g3] ASI_DMMU - .section .sun4v_1insn_patch, "ax" - .word 661b - stxa %g2, [%g3] ASI_MMU - .previous - - membar #Sync - add %g6, TI_FPREGS, %g2 - andcc %g1, FPRS_DL, %g0 - be,pn %icc, 4f - mov 0x40, %g3 - stda %f0, [%g2] ASI_BLK_S - stda %f16, [%g2 + %g3] ASI_BLK_S - andcc %g1, FPRS_DU, %g0 - be,pn %icc, 5f -4: add %g2, 128, %g2 - stda %f32, [%g2] ASI_BLK_S - stda %f48, [%g2 + %g3] ASI_BLK_S -5: mov SECONDARY_CONTEXT, %g1 - membar #Sync - -661: stxa %g5, [%g1] ASI_DMMU - .section .sun4v_1insn_patch, "ax" - .word 661b - stxa %g5, [%g1] ASI_MMU - .previous - - membar #Sync - ba,pt %xcc, etrap - wr %g0, 0, %fprs - - /* The registers for cross calls will be: - * - * DATA 0: [low 32-bits] Address of function to call, jmp to this - * [high 32-bits] MMU Context Argument 0, place in %g5 - * DATA 1: Address Argument 1, place in %g1 - * DATA 2: Address Argument 2, place in %g7 - * - * With this method we can do most of the cross-call tlb/cache - * flushing very quickly. - */ - .text - .align 32 - .globl do_ivec -do_ivec: - mov 0x40, %g3 - ldxa [%g3 + %g0] ASI_INTR_R, %g3 - sethi %hi(KERNBASE), %g4 - cmp %g3, %g4 - bgeu,pn %xcc, do_ivec_xcall - srlx %g3, 32, %g5 - stxa %g0, [%g0] ASI_INTR_RECEIVE - membar #Sync - - sethi %hi(ivector_table_pa), %g2 - ldx [%g2 + %lo(ivector_table_pa)], %g2 - sllx %g3, 4, %g3 - add %g2, %g3, %g3 - - TRAP_LOAD_IRQ_WORK_PA(%g6, %g1) - - ldx [%g6], %g5 - stxa %g5, [%g3] ASI_PHYS_USE_EC - stx %g3, [%g6] - wr %g0, 1 << PIL_DEVICE_IRQ, %set_softint - retry -do_ivec_xcall: - mov 0x50, %g1 - ldxa [%g1 + %g0] ASI_INTR_R, %g1 - srl %g3, 0, %g3 - - mov 0x60, %g7 - ldxa [%g7 + %g0] ASI_INTR_R, %g7 - stxa %g0, [%g0] ASI_INTR_RECEIVE - membar #Sync - ba,pt %xcc, 1f - nop - - .align 32 -1: jmpl %g3, %g0 - nop - - .globl getcc, setcc -getcc: - ldx [%o0 + PT_V9_TSTATE], %o1 - srlx %o1, 32, %o1 - and %o1, 0xf, %o1 - retl - stx %o1, [%o0 + PT_V9_G1] -setcc: - ldx [%o0 + PT_V9_TSTATE], %o1 - ldx [%o0 + PT_V9_G1], %o2 - or %g0, %ulo(TSTATE_ICC), %o3 - sllx %o3, 32, %o3 - andn %o1, %o3, %o1 - sllx %o2, 32, %o2 - and %o2, %o3, %o2 - or %o1, %o2, %o1 - retl - stx %o1, [%o0 + PT_V9_TSTATE] - - .globl utrap_trap -utrap_trap: /* %g3=handler,%g4=level */ - TRAP_LOAD_THREAD_REG(%g6, %g1) - ldx [%g6 + TI_UTRAPS], %g1 - brnz,pt %g1, invoke_utrap - nop - - ba,pt %xcc, etrap - rd %pc, %g7 - mov %l4, %o1 - call bad_trap - add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - clr %l6 - -invoke_utrap: - sllx %g3, 3, %g3 - ldx [%g1 + %g3], %g1 - save %sp, -128, %sp - rdpr %tstate, %l6 - rdpr %cwp, %l7 - andn %l6, TSTATE_CWP, %l6 - wrpr %l6, %l7, %tstate - rdpr %tpc, %l6 - rdpr %tnpc, %l7 - wrpr %g1, 0, %tnpc - done - - /* We need to carefully read the error status, ACK - * the errors, prevent recursive traps, and pass the - * information on to C code for logging. - * - * We pass the AFAR in as-is, and we encode the status - * information as described in asm-sparc64/sfafsr.h - */ - .globl __spitfire_access_error -__spitfire_access_error: - /* Disable ESTATE error reporting so that we do not - * take recursive traps and RED state the processor. - */ - stxa %g0, [%g0] ASI_ESTATE_ERROR_EN - membar #Sync - - mov UDBE_UE, %g1 - ldxa [%g0] ASI_AFSR, %g4 ! Get AFSR - - /* __spitfire_cee_trap branches here with AFSR in %g4 and - * UDBE_CE in %g1. It only clears ESTATE_ERR_CE in the - * ESTATE Error Enable register. - */ -__spitfire_cee_trap_continue: - ldxa [%g0] ASI_AFAR, %g5 ! Get AFAR - - rdpr %tt, %g3 - and %g3, 0x1ff, %g3 ! Paranoia - sllx %g3, SFSTAT_TRAP_TYPE_SHIFT, %g3 - or %g4, %g3, %g4 - rdpr %tl, %g3 - cmp %g3, 1 - mov 1, %g3 - bleu %xcc, 1f - sllx %g3, SFSTAT_TL_GT_ONE_SHIFT, %g3 - - or %g4, %g3, %g4 - - /* Read in the UDB error register state, clearing the - * sticky error bits as-needed. We only clear them if - * the UE bit is set. Likewise, __spitfire_cee_trap - * below will only do so if the CE bit is set. - * - * NOTE: UltraSparc-I/II have high and low UDB error - * registers, corresponding to the two UDB units - * present on those chips. UltraSparc-IIi only - * has a single UDB, called "SDB" in the manual. - * For IIi the upper UDB register always reads - * as zero so for our purposes things will just - * work with the checks below. - */ -1: ldxa [%g0] ASI_UDBH_ERROR_R, %g3 - and %g3, 0x3ff, %g7 ! Paranoia - sllx %g7, SFSTAT_UDBH_SHIFT, %g7 - or %g4, %g7, %g4 - andcc %g3, %g1, %g3 ! UDBE_UE or UDBE_CE - be,pn %xcc, 1f - nop - stxa %g3, [%g0] ASI_UDB_ERROR_W - membar #Sync - -1: mov 0x18, %g3 - ldxa [%g3] ASI_UDBL_ERROR_R, %g3 - and %g3, 0x3ff, %g7 ! Paranoia - sllx %g7, SFSTAT_UDBL_SHIFT, %g7 - or %g4, %g7, %g4 - andcc %g3, %g1, %g3 ! UDBE_UE or UDBE_CE - be,pn %xcc, 1f - nop - mov 0x18, %g7 - stxa %g3, [%g7] ASI_UDB_ERROR_W - membar #Sync - -1: /* Ok, now that we've latched the error state, - * clear the sticky bits in the AFSR. - */ - stxa %g4, [%g0] ASI_AFSR - membar #Sync - - rdpr %tl, %g2 - cmp %g2, 1 - rdpr %pil, %g2 - bleu,pt %xcc, 1f - wrpr %g0, 15, %pil - - ba,pt %xcc, etraptl1 - rd %pc, %g7 - - ba,pt %xcc, 2f - nop - -1: ba,pt %xcc, etrap_irq - rd %pc, %g7 - -2: -#ifdef CONFIG_TRACE_IRQFLAGS - call trace_hardirqs_off - nop -#endif - mov %l4, %o1 - mov %l5, %o2 - call spitfire_access_error - add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - clr %l6 - - /* This is the trap handler entry point for ECC correctable - * errors. They are corrected, but we listen for the trap - * so that the event can be logged. - * - * Disrupting errors are either: - * 1) single-bit ECC errors during UDB reads to system - * memory - * 2) data parity errors during write-back events - * - * As far as I can make out from the manual, the CEE trap - * is only for correctable errors during memory read - * accesses by the front-end of the processor. - * - * The code below is only for trap level 1 CEE events, - * as it is the only situation where we can safely record - * and log. For trap level >1 we just clear the CE bit - * in the AFSR and return. - * - * This is just like __spiftire_access_error above, but it - * specifically handles correctable errors. If an - * uncorrectable error is indicated in the AFSR we - * will branch directly above to __spitfire_access_error - * to handle it instead. Uncorrectable therefore takes - * priority over correctable, and the error logging - * C code will notice this case by inspecting the - * trap type. - */ - .globl __spitfire_cee_trap -__spitfire_cee_trap: - ldxa [%g0] ASI_AFSR, %g4 ! Get AFSR - mov 1, %g3 - sllx %g3, SFAFSR_UE_SHIFT, %g3 - andcc %g4, %g3, %g0 ! Check for UE - bne,pn %xcc, __spitfire_access_error - nop - - /* Ok, in this case we only have a correctable error. - * Indicate we only wish to capture that state in register - * %g1, and we only disable CE error reporting unlike UE - * handling which disables all errors. - */ - ldxa [%g0] ASI_ESTATE_ERROR_EN, %g3 - andn %g3, ESTATE_ERR_CE, %g3 - stxa %g3, [%g0] ASI_ESTATE_ERROR_EN - membar #Sync - - /* Preserve AFSR in %g4, indicate UDB state to capture in %g1 */ - ba,pt %xcc, __spitfire_cee_trap_continue - mov UDBE_CE, %g1 - - .globl __spitfire_data_access_exception - .globl __spitfire_data_access_exception_tl1 -__spitfire_data_access_exception_tl1: - rdpr %pstate, %g4 - wrpr %g4, PSTATE_MG|PSTATE_AG, %pstate - mov TLB_SFSR, %g3 - mov DMMU_SFAR, %g5 - ldxa [%g3] ASI_DMMU, %g4 ! Get SFSR - ldxa [%g5] ASI_DMMU, %g5 ! Get SFAR - stxa %g0, [%g3] ASI_DMMU ! Clear SFSR.FaultValid bit - membar #Sync - rdpr %tt, %g3 - cmp %g3, 0x80 ! first win spill/fill trap - blu,pn %xcc, 1f - cmp %g3, 0xff ! last win spill/fill trap - bgu,pn %xcc, 1f - nop - ba,pt %xcc, winfix_dax - rdpr %tpc, %g3 -1: sethi %hi(109f), %g7 - ba,pt %xcc, etraptl1 -109: or %g7, %lo(109b), %g7 - mov %l4, %o1 - mov %l5, %o2 - call spitfire_data_access_exception_tl1 - add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - clr %l6 - -__spitfire_data_access_exception: - rdpr %pstate, %g4 - wrpr %g4, PSTATE_MG|PSTATE_AG, %pstate - mov TLB_SFSR, %g3 - mov DMMU_SFAR, %g5 - ldxa [%g3] ASI_DMMU, %g4 ! Get SFSR - ldxa [%g5] ASI_DMMU, %g5 ! Get SFAR - stxa %g0, [%g3] ASI_DMMU ! Clear SFSR.FaultValid bit - membar #Sync - sethi %hi(109f), %g7 - ba,pt %xcc, etrap -109: or %g7, %lo(109b), %g7 - mov %l4, %o1 - mov %l5, %o2 - call spitfire_data_access_exception - add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - clr %l6 - - .globl __spitfire_insn_access_exception - .globl __spitfire_insn_access_exception_tl1 -__spitfire_insn_access_exception_tl1: - rdpr %pstate, %g4 - wrpr %g4, PSTATE_MG|PSTATE_AG, %pstate - mov TLB_SFSR, %g3 - ldxa [%g3] ASI_IMMU, %g4 ! Get SFSR - rdpr %tpc, %g5 ! IMMU has no SFAR, use TPC - stxa %g0, [%g3] ASI_IMMU ! Clear FaultValid bit - membar #Sync - sethi %hi(109f), %g7 - ba,pt %xcc, etraptl1 -109: or %g7, %lo(109b), %g7 - mov %l4, %o1 - mov %l5, %o2 - call spitfire_insn_access_exception_tl1 - add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - clr %l6 - -__spitfire_insn_access_exception: - rdpr %pstate, %g4 - wrpr %g4, PSTATE_MG|PSTATE_AG, %pstate - mov TLB_SFSR, %g3 - ldxa [%g3] ASI_IMMU, %g4 ! Get SFSR - rdpr %tpc, %g5 ! IMMU has no SFAR, use TPC - stxa %g0, [%g3] ASI_IMMU ! Clear FaultValid bit - membar #Sync - sethi %hi(109f), %g7 - ba,pt %xcc, etrap -109: or %g7, %lo(109b), %g7 - mov %l4, %o1 - mov %l5, %o2 - call spitfire_insn_access_exception - add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - clr %l6 - - /* These get patched into the trap table at boot time - * once we know we have a cheetah processor. - */ - .globl cheetah_fecc_trap_vector, cheetah_fecc_trap_vector_tl1 -cheetah_fecc_trap_vector: - membar #Sync - ldxa [%g0] ASI_DCU_CONTROL_REG, %g1 - andn %g1, DCU_DC | DCU_IC, %g1 - stxa %g1, [%g0] ASI_DCU_CONTROL_REG - membar #Sync - sethi %hi(cheetah_fast_ecc), %g2 - jmpl %g2 + %lo(cheetah_fast_ecc), %g0 - mov 0, %g1 -cheetah_fecc_trap_vector_tl1: - membar #Sync - ldxa [%g0] ASI_DCU_CONTROL_REG, %g1 - andn %g1, DCU_DC | DCU_IC, %g1 - stxa %g1, [%g0] ASI_DCU_CONTROL_REG - membar #Sync - sethi %hi(cheetah_fast_ecc), %g2 - jmpl %g2 + %lo(cheetah_fast_ecc), %g0 - mov 1, %g1 - .globl cheetah_cee_trap_vector, cheetah_cee_trap_vector_tl1 -cheetah_cee_trap_vector: - membar #Sync - ldxa [%g0] ASI_DCU_CONTROL_REG, %g1 - andn %g1, DCU_IC, %g1 - stxa %g1, [%g0] ASI_DCU_CONTROL_REG - membar #Sync - sethi %hi(cheetah_cee), %g2 - jmpl %g2 + %lo(cheetah_cee), %g0 - mov 0, %g1 -cheetah_cee_trap_vector_tl1: - membar #Sync - ldxa [%g0] ASI_DCU_CONTROL_REG, %g1 - andn %g1, DCU_IC, %g1 - stxa %g1, [%g0] ASI_DCU_CONTROL_REG - membar #Sync - sethi %hi(cheetah_cee), %g2 - jmpl %g2 + %lo(cheetah_cee), %g0 - mov 1, %g1 - .globl cheetah_deferred_trap_vector, cheetah_deferred_trap_vector_tl1 -cheetah_deferred_trap_vector: - membar #Sync - ldxa [%g0] ASI_DCU_CONTROL_REG, %g1; - andn %g1, DCU_DC | DCU_IC, %g1; - stxa %g1, [%g0] ASI_DCU_CONTROL_REG; - membar #Sync; - sethi %hi(cheetah_deferred_trap), %g2 - jmpl %g2 + %lo(cheetah_deferred_trap), %g0 - mov 0, %g1 -cheetah_deferred_trap_vector_tl1: - membar #Sync; - ldxa [%g0] ASI_DCU_CONTROL_REG, %g1; - andn %g1, DCU_DC | DCU_IC, %g1; - stxa %g1, [%g0] ASI_DCU_CONTROL_REG; - membar #Sync; - sethi %hi(cheetah_deferred_trap), %g2 - jmpl %g2 + %lo(cheetah_deferred_trap), %g0 - mov 1, %g1 - - /* Cheetah+ specific traps. These are for the new I/D cache parity - * error traps. The first argument to cheetah_plus_parity_handler - * is encoded as follows: - * - * Bit0: 0=dcache,1=icache - * Bit1: 0=recoverable,1=unrecoverable - */ - .globl cheetah_plus_dcpe_trap_vector, cheetah_plus_dcpe_trap_vector_tl1 -cheetah_plus_dcpe_trap_vector: - membar #Sync - sethi %hi(do_cheetah_plus_data_parity), %g7 - jmpl %g7 + %lo(do_cheetah_plus_data_parity), %g0 - nop - nop - nop - nop - nop - -do_cheetah_plus_data_parity: - rdpr %pil, %g2 - wrpr %g0, 15, %pil - ba,pt %xcc, etrap_irq - rd %pc, %g7 -#ifdef CONFIG_TRACE_IRQFLAGS - call trace_hardirqs_off - nop -#endif - mov 0x0, %o0 - call cheetah_plus_parity_error - add %sp, PTREGS_OFF, %o1 - ba,a,pt %xcc, rtrap_irq - -cheetah_plus_dcpe_trap_vector_tl1: - membar #Sync - wrpr PSTATE_IG | PSTATE_PEF | PSTATE_PRIV, %pstate - sethi %hi(do_dcpe_tl1), %g3 - jmpl %g3 + %lo(do_dcpe_tl1), %g0 - nop - nop - nop - nop - - .globl cheetah_plus_icpe_trap_vector, cheetah_plus_icpe_trap_vector_tl1 -cheetah_plus_icpe_trap_vector: - membar #Sync - sethi %hi(do_cheetah_plus_insn_parity), %g7 - jmpl %g7 + %lo(do_cheetah_plus_insn_parity), %g0 - nop - nop - nop - nop - nop - -do_cheetah_plus_insn_parity: - rdpr %pil, %g2 - wrpr %g0, 15, %pil - ba,pt %xcc, etrap_irq - rd %pc, %g7 -#ifdef CONFIG_TRACE_IRQFLAGS - call trace_hardirqs_off - nop -#endif - mov 0x1, %o0 - call cheetah_plus_parity_error - add %sp, PTREGS_OFF, %o1 - ba,a,pt %xcc, rtrap_irq - -cheetah_plus_icpe_trap_vector_tl1: - membar #Sync - wrpr PSTATE_IG | PSTATE_PEF | PSTATE_PRIV, %pstate - sethi %hi(do_icpe_tl1), %g3 - jmpl %g3 + %lo(do_icpe_tl1), %g0 - nop - nop - nop - nop - - /* If we take one of these traps when tl >= 1, then we - * jump to interrupt globals. If some trap level above us - * was also using interrupt globals, we cannot recover. - * We may use all interrupt global registers except %g6. - */ - .globl do_dcpe_tl1, do_icpe_tl1 -do_dcpe_tl1: - rdpr %tl, %g1 ! Save original trap level - mov 1, %g2 ! Setup TSTATE checking loop - sethi %hi(TSTATE_IG), %g3 ! TSTATE mask bit -1: wrpr %g2, %tl ! Set trap level to check - rdpr %tstate, %g4 ! Read TSTATE for this level - andcc %g4, %g3, %g0 ! Interrupt globals in use? - bne,a,pn %xcc, do_dcpe_tl1_fatal ! Yep, irrecoverable - wrpr %g1, %tl ! Restore original trap level - add %g2, 1, %g2 ! Next trap level - cmp %g2, %g1 ! Hit them all yet? - ble,pt %icc, 1b ! Not yet - nop - wrpr %g1, %tl ! Restore original trap level -do_dcpe_tl1_nonfatal: /* Ok we may use interrupt globals safely. */ - sethi %hi(dcache_parity_tl1_occurred), %g2 - lduw [%g2 + %lo(dcache_parity_tl1_occurred)], %g1 - add %g1, 1, %g1 - stw %g1, [%g2 + %lo(dcache_parity_tl1_occurred)] - /* Reset D-cache parity */ - sethi %hi(1 << 16), %g1 ! D-cache size - mov (1 << 5), %g2 ! D-cache line size - sub %g1, %g2, %g1 ! Move down 1 cacheline -1: srl %g1, 14, %g3 ! Compute UTAG - membar #Sync - stxa %g3, [%g1] ASI_DCACHE_UTAG - membar #Sync - sub %g2, 8, %g3 ! 64-bit data word within line -2: membar #Sync - stxa %g0, [%g1 + %g3] ASI_DCACHE_DATA - membar #Sync - subcc %g3, 8, %g3 ! Next 64-bit data word - bge,pt %icc, 2b - nop - subcc %g1, %g2, %g1 ! Next cacheline - bge,pt %icc, 1b - nop - ba,pt %xcc, dcpe_icpe_tl1_common - nop - -do_dcpe_tl1_fatal: - sethi %hi(1f), %g7 - ba,pt %xcc, etraptl1 -1: or %g7, %lo(1b), %g7 - mov 0x2, %o0 - call cheetah_plus_parity_error - add %sp, PTREGS_OFF, %o1 - ba,pt %xcc, rtrap - clr %l6 - -do_icpe_tl1: - rdpr %tl, %g1 ! Save original trap level - mov 1, %g2 ! Setup TSTATE checking loop - sethi %hi(TSTATE_IG), %g3 ! TSTATE mask bit -1: wrpr %g2, %tl ! Set trap level to check - rdpr %tstate, %g4 ! Read TSTATE for this level - andcc %g4, %g3, %g0 ! Interrupt globals in use? - bne,a,pn %xcc, do_icpe_tl1_fatal ! Yep, irrecoverable - wrpr %g1, %tl ! Restore original trap level - add %g2, 1, %g2 ! Next trap level - cmp %g2, %g1 ! Hit them all yet? - ble,pt %icc, 1b ! Not yet - nop - wrpr %g1, %tl ! Restore original trap level -do_icpe_tl1_nonfatal: /* Ok we may use interrupt globals safely. */ - sethi %hi(icache_parity_tl1_occurred), %g2 - lduw [%g2 + %lo(icache_parity_tl1_occurred)], %g1 - add %g1, 1, %g1 - stw %g1, [%g2 + %lo(icache_parity_tl1_occurred)] - /* Flush I-cache */ - sethi %hi(1 << 15), %g1 ! I-cache size - mov (1 << 5), %g2 ! I-cache line size - sub %g1, %g2, %g1 -1: or %g1, (2 << 3), %g3 - stxa %g0, [%g3] ASI_IC_TAG - membar #Sync - subcc %g1, %g2, %g1 - bge,pt %icc, 1b - nop - ba,pt %xcc, dcpe_icpe_tl1_common - nop - -do_icpe_tl1_fatal: - sethi %hi(1f), %g7 - ba,pt %xcc, etraptl1 -1: or %g7, %lo(1b), %g7 - mov 0x3, %o0 - call cheetah_plus_parity_error - add %sp, PTREGS_OFF, %o1 - ba,pt %xcc, rtrap - clr %l6 - -dcpe_icpe_tl1_common: - /* Flush D-cache, re-enable D/I caches in DCU and finally - * retry the trapping instruction. - */ - sethi %hi(1 << 16), %g1 ! D-cache size - mov (1 << 5), %g2 ! D-cache line size - sub %g1, %g2, %g1 -1: stxa %g0, [%g1] ASI_DCACHE_TAG - membar #Sync - subcc %g1, %g2, %g1 - bge,pt %icc, 1b - nop - ldxa [%g0] ASI_DCU_CONTROL_REG, %g1 - or %g1, (DCU_DC | DCU_IC), %g1 - stxa %g1, [%g0] ASI_DCU_CONTROL_REG - membar #Sync - retry - - /* Capture I/D/E-cache state into per-cpu error scoreboard. - * - * %g1: (TL>=0) ? 1 : 0 - * %g2: scratch - * %g3: scratch - * %g4: AFSR - * %g5: AFAR - * %g6: unused, will have current thread ptr after etrap - * %g7: scratch - */ -__cheetah_log_error: - /* Put "TL1" software bit into AFSR. */ - and %g1, 0x1, %g1 - sllx %g1, 63, %g2 - or %g4, %g2, %g4 - - /* Get log entry pointer for this cpu at this trap level. */ - BRANCH_IF_JALAPENO(g2,g3,50f) - ldxa [%g0] ASI_SAFARI_CONFIG, %g2 - srlx %g2, 17, %g2 - ba,pt %xcc, 60f - and %g2, 0x3ff, %g2 - -50: ldxa [%g0] ASI_JBUS_CONFIG, %g2 - srlx %g2, 17, %g2 - and %g2, 0x1f, %g2 - -60: sllx %g2, 9, %g2 - sethi %hi(cheetah_error_log), %g3 - ldx [%g3 + %lo(cheetah_error_log)], %g3 - brz,pn %g3, 80f - nop - - add %g3, %g2, %g3 - sllx %g1, 8, %g1 - add %g3, %g1, %g1 - - /* %g1 holds pointer to the top of the logging scoreboard */ - ldx [%g1 + 0x0], %g7 - cmp %g7, -1 - bne,pn %xcc, 80f - nop - - stx %g4, [%g1 + 0x0] - stx %g5, [%g1 + 0x8] - add %g1, 0x10, %g1 - - /* %g1 now points to D-cache logging area */ - set 0x3ff8, %g2 /* DC_addr mask */ - and %g5, %g2, %g2 /* DC_addr bits of AFAR */ - srlx %g5, 12, %g3 - or %g3, 1, %g3 /* PHYS tag + valid */ - -10: ldxa [%g2] ASI_DCACHE_TAG, %g7 - cmp %g3, %g7 /* TAG match? */ - bne,pt %xcc, 13f - nop - - /* Yep, what we want, capture state. */ - stx %g2, [%g1 + 0x20] - stx %g7, [%g1 + 0x28] - - /* A membar Sync is required before and after utag access. */ - membar #Sync - ldxa [%g2] ASI_DCACHE_UTAG, %g7 - membar #Sync - stx %g7, [%g1 + 0x30] - ldxa [%g2] ASI_DCACHE_SNOOP_TAG, %g7 - stx %g7, [%g1 + 0x38] - clr %g3 - -12: ldxa [%g2 + %g3] ASI_DCACHE_DATA, %g7 - stx %g7, [%g1] - add %g3, (1 << 5), %g3 - cmp %g3, (4 << 5) - bl,pt %xcc, 12b - add %g1, 0x8, %g1 - - ba,pt %xcc, 20f - add %g1, 0x20, %g1 - -13: sethi %hi(1 << 14), %g7 - add %g2, %g7, %g2 - srlx %g2, 14, %g7 - cmp %g7, 4 - bl,pt %xcc, 10b - nop - - add %g1, 0x40, %g1 - - /* %g1 now points to I-cache logging area */ -20: set 0x1fe0, %g2 /* IC_addr mask */ - and %g5, %g2, %g2 /* IC_addr bits of AFAR */ - sllx %g2, 1, %g2 /* IC_addr[13:6]==VA[12:5] */ - srlx %g5, (13 - 8), %g3 /* Make PTAG */ - andn %g3, 0xff, %g3 /* Mask off undefined bits */ - -21: ldxa [%g2] ASI_IC_TAG, %g7 - andn %g7, 0xff, %g7 - cmp %g3, %g7 - bne,pt %xcc, 23f - nop - - /* Yep, what we want, capture state. */ - stx %g2, [%g1 + 0x40] - stx %g7, [%g1 + 0x48] - add %g2, (1 << 3), %g2 - ldxa [%g2] ASI_IC_TAG, %g7 - add %g2, (1 << 3), %g2 - stx %g7, [%g1 + 0x50] - ldxa [%g2] ASI_IC_TAG, %g7 - add %g2, (1 << 3), %g2 - stx %g7, [%g1 + 0x60] - ldxa [%g2] ASI_IC_TAG, %g7 - stx %g7, [%g1 + 0x68] - sub %g2, (3 << 3), %g2 - ldxa [%g2] ASI_IC_STAG, %g7 - stx %g7, [%g1 + 0x58] - clr %g3 - srlx %g2, 2, %g2 - -22: ldxa [%g2 + %g3] ASI_IC_INSTR, %g7 - stx %g7, [%g1] - add %g3, (1 << 3), %g3 - cmp %g3, (8 << 3) - bl,pt %xcc, 22b - add %g1, 0x8, %g1 - - ba,pt %xcc, 30f - add %g1, 0x30, %g1 - -23: sethi %hi(1 << 14), %g7 - add %g2, %g7, %g2 - srlx %g2, 14, %g7 - cmp %g7, 4 - bl,pt %xcc, 21b - nop - - add %g1, 0x70, %g1 - - /* %g1 now points to E-cache logging area */ -30: andn %g5, (32 - 1), %g2 - stx %g2, [%g1 + 0x20] - ldxa [%g2] ASI_EC_TAG_DATA, %g7 - stx %g7, [%g1 + 0x28] - ldxa [%g2] ASI_EC_R, %g0 - clr %g3 - -31: ldxa [%g3] ASI_EC_DATA, %g7 - stx %g7, [%g1 + %g3] - add %g3, 0x8, %g3 - cmp %g3, 0x20 - - bl,pt %xcc, 31b - nop -80: - rdpr %tt, %g2 - cmp %g2, 0x70 - be c_fast_ecc - cmp %g2, 0x63 - be c_cee - nop - ba,pt %xcc, c_deferred - - /* Cheetah FECC trap handling, we get here from tl{0,1}_fecc - * in the trap table. That code has done a memory barrier - * and has disabled both the I-cache and D-cache in the DCU - * control register. The I-cache is disabled so that we may - * capture the corrupted cache line, and the D-cache is disabled - * because corrupt data may have been placed there and we don't - * want to reference it. - * - * %g1 is one if this trap occurred at %tl >= 1. - * - * Next, we turn off error reporting so that we don't recurse. - */ - .globl cheetah_fast_ecc -cheetah_fast_ecc: - ldxa [%g0] ASI_ESTATE_ERROR_EN, %g2 - andn %g2, ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN, %g2 - stxa %g2, [%g0] ASI_ESTATE_ERROR_EN - membar #Sync - - /* Fetch and clear AFSR/AFAR */ - ldxa [%g0] ASI_AFSR, %g4 - ldxa [%g0] ASI_AFAR, %g5 - stxa %g4, [%g0] ASI_AFSR - membar #Sync - - ba,pt %xcc, __cheetah_log_error - nop - -c_fast_ecc: - rdpr %pil, %g2 - wrpr %g0, 15, %pil - ba,pt %xcc, etrap_irq - rd %pc, %g7 -#ifdef CONFIG_TRACE_IRQFLAGS - call trace_hardirqs_off - nop -#endif - mov %l4, %o1 - mov %l5, %o2 - call cheetah_fecc_handler - add %sp, PTREGS_OFF, %o0 - ba,a,pt %xcc, rtrap_irq - - /* Our caller has disabled I-cache and performed membar Sync. */ - .globl cheetah_cee -cheetah_cee: - ldxa [%g0] ASI_ESTATE_ERROR_EN, %g2 - andn %g2, ESTATE_ERROR_CEEN, %g2 - stxa %g2, [%g0] ASI_ESTATE_ERROR_EN - membar #Sync - - /* Fetch and clear AFSR/AFAR */ - ldxa [%g0] ASI_AFSR, %g4 - ldxa [%g0] ASI_AFAR, %g5 - stxa %g4, [%g0] ASI_AFSR - membar #Sync - - ba,pt %xcc, __cheetah_log_error - nop - -c_cee: - rdpr %pil, %g2 - wrpr %g0, 15, %pil - ba,pt %xcc, etrap_irq - rd %pc, %g7 -#ifdef CONFIG_TRACE_IRQFLAGS - call trace_hardirqs_off - nop -#endif - mov %l4, %o1 - mov %l5, %o2 - call cheetah_cee_handler - add %sp, PTREGS_OFF, %o0 - ba,a,pt %xcc, rtrap_irq - - /* Our caller has disabled I-cache+D-cache and performed membar Sync. */ - .globl cheetah_deferred_trap -cheetah_deferred_trap: - ldxa [%g0] ASI_ESTATE_ERROR_EN, %g2 - andn %g2, ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN, %g2 - stxa %g2, [%g0] ASI_ESTATE_ERROR_EN - membar #Sync - - /* Fetch and clear AFSR/AFAR */ - ldxa [%g0] ASI_AFSR, %g4 - ldxa [%g0] ASI_AFAR, %g5 - stxa %g4, [%g0] ASI_AFSR - membar #Sync - - ba,pt %xcc, __cheetah_log_error - nop - -c_deferred: - rdpr %pil, %g2 - wrpr %g0, 15, %pil - ba,pt %xcc, etrap_irq - rd %pc, %g7 -#ifdef CONFIG_TRACE_IRQFLAGS - call trace_hardirqs_off - nop -#endif - mov %l4, %o1 - mov %l5, %o2 - call cheetah_deferred_handler - add %sp, PTREGS_OFF, %o0 - ba,a,pt %xcc, rtrap_irq - - .globl __do_privact -__do_privact: - mov TLB_SFSR, %g3 - stxa %g0, [%g3] ASI_DMMU ! Clear FaultValid bit - membar #Sync - sethi %hi(109f), %g7 - ba,pt %xcc, etrap -109: or %g7, %lo(109b), %g7 - call do_privact - add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - clr %l6 - - .globl do_mna -do_mna: - rdpr %tl, %g3 - cmp %g3, 1 - - /* Setup %g4/%g5 now as they are used in the - * winfixup code. - */ - mov TLB_SFSR, %g3 - mov DMMU_SFAR, %g4 - ldxa [%g4] ASI_DMMU, %g4 - ldxa [%g3] ASI_DMMU, %g5 - stxa %g0, [%g3] ASI_DMMU ! Clear FaultValid bit - membar #Sync - bgu,pn %icc, winfix_mna - rdpr %tpc, %g3 - -1: sethi %hi(109f), %g7 - ba,pt %xcc, etrap -109: or %g7, %lo(109b), %g7 - mov %l4, %o1 - mov %l5, %o2 - call mem_address_unaligned - add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - clr %l6 - - .globl do_lddfmna -do_lddfmna: - sethi %hi(109f), %g7 - mov TLB_SFSR, %g4 - ldxa [%g4] ASI_DMMU, %g5 - stxa %g0, [%g4] ASI_DMMU ! Clear FaultValid bit - membar #Sync - mov DMMU_SFAR, %g4 - ldxa [%g4] ASI_DMMU, %g4 - ba,pt %xcc, etrap -109: or %g7, %lo(109b), %g7 - mov %l4, %o1 - mov %l5, %o2 - call handle_lddfmna - add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - clr %l6 - - .globl do_stdfmna -do_stdfmna: - sethi %hi(109f), %g7 - mov TLB_SFSR, %g4 - ldxa [%g4] ASI_DMMU, %g5 - stxa %g0, [%g4] ASI_DMMU ! Clear FaultValid bit - membar #Sync - mov DMMU_SFAR, %g4 - ldxa [%g4] ASI_DMMU, %g4 - ba,pt %xcc, etrap -109: or %g7, %lo(109b), %g7 - mov %l4, %o1 - mov %l5, %o2 - call handle_stdfmna - add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - clr %l6 - - .globl breakpoint_trap -breakpoint_trap: - call sparc_breakpoint - add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - nop - - /* SunOS's execv() call only specifies the argv argument, the - * environment settings are the same as the calling processes. - */ - .globl sunos_execv -sys_execve: - sethi %hi(sparc_execve), %g1 - ba,pt %xcc, execve_merge - or %g1, %lo(sparc_execve), %g1 -#ifdef CONFIG_COMPAT - .globl sys_execve -sunos_execv: - stx %g0, [%sp + PTREGS_OFF + PT_V9_I2] - .globl sys32_execve -sys32_execve: - sethi %hi(sparc32_execve), %g1 - or %g1, %lo(sparc32_execve), %g1 -#endif -execve_merge: - flushw - jmpl %g1, %g0 - add %sp, PTREGS_OFF, %o0 - - .globl sys_pipe, sys_sigpause, sys_nis_syscall - .globl sys_rt_sigreturn - .globl sys_ptrace - .globl sys_sigaltstack - .align 32 -sys_pipe: ba,pt %xcc, sparc_pipe - add %sp, PTREGS_OFF, %o0 -sys_nis_syscall:ba,pt %xcc, c_sys_nis_syscall - add %sp, PTREGS_OFF, %o0 -sys_memory_ordering: - ba,pt %xcc, sparc_memory_ordering - add %sp, PTREGS_OFF, %o1 -sys_sigaltstack:ba,pt %xcc, do_sigaltstack - add %i6, STACK_BIAS, %o2 -#ifdef CONFIG_COMPAT - .globl sys32_sigstack -sys32_sigstack: ba,pt %xcc, do_sys32_sigstack - mov %i6, %o2 - .globl sys32_sigaltstack -sys32_sigaltstack: - ba,pt %xcc, do_sys32_sigaltstack - mov %i6, %o2 -#endif - .align 32 -#ifdef CONFIG_COMPAT - .globl sys32_sigreturn -sys32_sigreturn: - add %sp, PTREGS_OFF, %o0 - call do_sigreturn32 - add %o7, 1f-.-4, %o7 - nop -#endif -sys_rt_sigreturn: - add %sp, PTREGS_OFF, %o0 - call do_rt_sigreturn - add %o7, 1f-.-4, %o7 - nop -#ifdef CONFIG_COMPAT - .globl sys32_rt_sigreturn -sys32_rt_sigreturn: - add %sp, PTREGS_OFF, %o0 - call do_rt_sigreturn32 - add %o7, 1f-.-4, %o7 - nop -#endif - .align 32 -1: ldx [%curptr + TI_FLAGS], %l5 - andcc %l5, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT), %g0 - be,pt %icc, rtrap - clr %l6 - add %sp, PTREGS_OFF, %o0 - call syscall_trace - mov 1, %o1 - - ba,pt %xcc, rtrap - clr %l6 - - /* This is how fork() was meant to be done, 8 instruction entry. - * - * I questioned the following code briefly, let me clear things - * up so you must not reason on it like I did. - * - * Know the fork_kpsr etc. we use in the sparc32 port? We don't - * need it here because the only piece of window state we copy to - * the child is the CWP register. Even if the parent sleeps, - * we are safe because we stuck it into pt_regs of the parent - * so it will not change. - * - * XXX This raises the question, whether we can do the same on - * XXX sparc32 to get rid of fork_kpsr _and_ fork_kwim. The - * XXX answer is yes. We stick fork_kpsr in UREG_G0 and - * XXX fork_kwim in UREG_G1 (global registers are considered - * XXX volatile across a system call in the sparc ABI I think - * XXX if it isn't we can use regs->y instead, anyone who depends - * XXX upon the Y register being preserved across a fork deserves - * XXX to lose). - * - * In fact we should take advantage of that fact for other things - * during system calls... - */ - .globl sys_fork, sys_vfork, sys_clone, sparc_exit - .globl ret_from_syscall - .align 32 -sys_vfork: /* Under Linux, vfork and fork are just special cases of clone. */ - sethi %hi(0x4000 | 0x0100 | SIGCHLD), %o0 - or %o0, %lo(0x4000 | 0x0100 | SIGCHLD), %o0 - ba,pt %xcc, sys_clone -sys_fork: clr %o1 - mov SIGCHLD, %o0 -sys_clone: flushw - movrz %o1, %fp, %o1 - mov 0, %o3 - ba,pt %xcc, sparc_do_fork - add %sp, PTREGS_OFF, %o2 -ret_from_syscall: - /* Clear current_thread_info()->new_child, and - * check performance counter stuff too. - */ - stb %g0, [%g6 + TI_NEW_CHILD] - ldx [%g6 + TI_FLAGS], %l0 - call schedule_tail - mov %g7, %o0 - andcc %l0, _TIF_PERFCTR, %g0 - be,pt %icc, 1f - nop - ldx [%g6 + TI_PCR], %o7 - wr %g0, %o7, %pcr - - /* Blackbird errata workaround. See commentary in - * smp.c:smp_percpu_timer_interrupt() for more - * information. - */ - ba,pt %xcc, 99f - nop - .align 64 -99: wr %g0, %g0, %pic - rd %pic, %g0 - -1: b,pt %xcc, ret_sys_call - ldx [%sp + PTREGS_OFF + PT_V9_I0], %o0 -sparc_exit: rdpr %pstate, %g2 - wrpr %g2, PSTATE_IE, %pstate - rdpr %otherwin, %g1 - rdpr %cansave, %g3 - add %g3, %g1, %g3 - wrpr %g3, 0x0, %cansave - wrpr %g0, 0x0, %otherwin - wrpr %g2, 0x0, %pstate - ba,pt %xcc, sys_exit - stb %g0, [%g6 + TI_WSAVED] - -linux_sparc_ni_syscall: - sethi %hi(sys_ni_syscall), %l7 - b,pt %xcc, 4f - or %l7, %lo(sys_ni_syscall), %l7 - -linux_syscall_trace32: - add %sp, PTREGS_OFF, %o0 - call syscall_trace - clr %o1 - srl %i0, 0, %o0 - srl %i4, 0, %o4 - srl %i1, 0, %o1 - srl %i2, 0, %o2 - b,pt %xcc, 2f - srl %i3, 0, %o3 - -linux_syscall_trace: - add %sp, PTREGS_OFF, %o0 - call syscall_trace - clr %o1 - mov %i0, %o0 - mov %i1, %o1 - mov %i2, %o2 - mov %i3, %o3 - b,pt %xcc, 2f - mov %i4, %o4 - - - /* Linux 32-bit system calls enter here... */ - .align 32 - .globl linux_sparc_syscall32 -linux_sparc_syscall32: - /* Direct access to user regs, much faster. */ - cmp %g1, NR_SYSCALLS ! IEU1 Group - bgeu,pn %xcc, linux_sparc_ni_syscall ! CTI - srl %i0, 0, %o0 ! IEU0 - sll %g1, 2, %l4 ! IEU0 Group - srl %i4, 0, %o4 ! IEU1 - lduw [%l7 + %l4], %l7 ! Load - srl %i1, 0, %o1 ! IEU0 Group - ldx [%curptr + TI_FLAGS], %l0 ! Load - - srl %i5, 0, %o5 ! IEU1 - srl %i2, 0, %o2 ! IEU0 Group - andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT), %g0 - bne,pn %icc, linux_syscall_trace32 ! CTI - mov %i0, %l5 ! IEU1 - call %l7 ! CTI Group brk forced - srl %i3, 0, %o3 ! IEU0 - ba,a,pt %xcc, 3f - - /* Linux native system calls enter here... */ - .align 32 - .globl linux_sparc_syscall, ret_sys_call -linux_sparc_syscall: - /* Direct access to user regs, much faster. */ - cmp %g1, NR_SYSCALLS ! IEU1 Group - bgeu,pn %xcc, linux_sparc_ni_syscall ! CTI - mov %i0, %o0 ! IEU0 - sll %g1, 2, %l4 ! IEU0 Group - mov %i1, %o1 ! IEU1 - lduw [%l7 + %l4], %l7 ! Load -4: mov %i2, %o2 ! IEU0 Group - ldx [%curptr + TI_FLAGS], %l0 ! Load - - mov %i3, %o3 ! IEU1 - mov %i4, %o4 ! IEU0 Group - andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT), %g0 - bne,pn %icc, linux_syscall_trace ! CTI Group - mov %i0, %l5 ! IEU0 -2: call %l7 ! CTI Group brk forced - mov %i5, %o5 ! IEU0 - nop - -3: stx %o0, [%sp + PTREGS_OFF + PT_V9_I0] -ret_sys_call: - ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %g3 - ldx [%sp + PTREGS_OFF + PT_V9_TNPC], %l1 ! pc = npc - sra %o0, 0, %o0 - mov %ulo(TSTATE_XCARRY | TSTATE_ICARRY), %g2 - sllx %g2, 32, %g2 - - /* Check if force_successful_syscall_return() - * was invoked. - */ - ldub [%curptr + TI_SYS_NOERROR], %l2 - brnz,a,pn %l2, 80f - stb %g0, [%curptr + TI_SYS_NOERROR] - - cmp %o0, -ERESTART_RESTARTBLOCK - bgeu,pn %xcc, 1f - andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT), %l6 -80: - /* System call success, clear Carry condition code. */ - andn %g3, %g2, %g3 - stx %g3, [%sp + PTREGS_OFF + PT_V9_TSTATE] - bne,pn %icc, linux_syscall_trace2 - add %l1, 0x4, %l2 ! npc = npc+4 - stx %l1, [%sp + PTREGS_OFF + PT_V9_TPC] - ba,pt %xcc, rtrap_clr_l6 - stx %l2, [%sp + PTREGS_OFF + PT_V9_TNPC] - -1: - /* System call failure, set Carry condition code. - * Also, get abs(errno) to return to the process. - */ - andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT), %l6 - sub %g0, %o0, %o0 - or %g3, %g2, %g3 - stx %o0, [%sp + PTREGS_OFF + PT_V9_I0] - mov 1, %l6 - stx %g3, [%sp + PTREGS_OFF + PT_V9_TSTATE] - bne,pn %icc, linux_syscall_trace2 - add %l1, 0x4, %l2 ! npc = npc+4 - stx %l1, [%sp + PTREGS_OFF + PT_V9_TPC] - - b,pt %xcc, rtrap - stx %l2, [%sp + PTREGS_OFF + PT_V9_TNPC] -linux_syscall_trace2: - add %sp, PTREGS_OFF, %o0 - call syscall_trace - mov 1, %o1 - stx %l1, [%sp + PTREGS_OFF + PT_V9_TPC] - ba,pt %xcc, rtrap - stx %l2, [%sp + PTREGS_OFF + PT_V9_TNPC] - - .align 32 - .globl __flushw_user -__flushw_user: - rdpr %otherwin, %g1 - brz,pn %g1, 2f - clr %g2 -1: save %sp, -128, %sp - rdpr %otherwin, %g1 - brnz,pt %g1, 1b - add %g2, 1, %g2 -1: sub %g2, 1, %g2 - brnz,pt %g2, 1b - restore %g0, %g0, %g0 -2: retl - nop - - /* Flush %fp and %i7 to the stack for all register - * windows active inside of the cpu. This allows - * show_stack_trace() to avoid using an expensive - * 'flushw'. - */ - .globl stack_trace_flush - .type stack_trace_flush,#function -stack_trace_flush: - rdpr %pstate, %o0 - wrpr %o0, PSTATE_IE, %pstate - - rdpr %cwp, %g1 - rdpr %canrestore, %g2 - sub %g1, 1, %g3 - -1: brz,pn %g2, 2f - sub %g2, 1, %g2 - wrpr %g3, %cwp - stx %fp, [%sp + STACK_BIAS + RW_V9_I6] - stx %i7, [%sp + STACK_BIAS + RW_V9_I7] - ba,pt %xcc, 1b - sub %g3, 1, %g3 - -2: wrpr %g1, %cwp - wrpr %o0, %pstate - - retl - nop - .size stack_trace_flush,.-stack_trace_flush - -#ifdef CONFIG_SMP - .globl hard_smp_processor_id -hard_smp_processor_id: -#endif - .globl real_hard_smp_processor_id -real_hard_smp_processor_id: - __GET_CPUID(%o0) - retl - nop - - /* %o0: devhandle - * %o1: devino - * - * returns %o0: sysino - */ - .globl sun4v_devino_to_sysino - .type sun4v_devino_to_sysino,#function -sun4v_devino_to_sysino: - mov HV_FAST_INTR_DEVINO2SYSINO, %o5 - ta HV_FAST_TRAP - retl - mov %o1, %o0 - .size sun4v_devino_to_sysino, .-sun4v_devino_to_sysino - - /* %o0: sysino - * - * returns %o0: intr_enabled (HV_INTR_{DISABLED,ENABLED}) - */ - .globl sun4v_intr_getenabled - .type sun4v_intr_getenabled,#function -sun4v_intr_getenabled: - mov HV_FAST_INTR_GETENABLED, %o5 - ta HV_FAST_TRAP - retl - mov %o1, %o0 - .size sun4v_intr_getenabled, .-sun4v_intr_getenabled - - /* %o0: sysino - * %o1: intr_enabled (HV_INTR_{DISABLED,ENABLED}) - */ - .globl sun4v_intr_setenabled - .type sun4v_intr_setenabled,#function -sun4v_intr_setenabled: - mov HV_FAST_INTR_SETENABLED, %o5 - ta HV_FAST_TRAP - retl - nop - .size sun4v_intr_setenabled, .-sun4v_intr_setenabled - - /* %o0: sysino - * - * returns %o0: intr_state (HV_INTR_STATE_*) - */ - .globl sun4v_intr_getstate - .type sun4v_intr_getstate,#function -sun4v_intr_getstate: - mov HV_FAST_INTR_GETSTATE, %o5 - ta HV_FAST_TRAP - retl - mov %o1, %o0 - .size sun4v_intr_getstate, .-sun4v_intr_getstate - - /* %o0: sysino - * %o1: intr_state (HV_INTR_STATE_*) - */ - .globl sun4v_intr_setstate - .type sun4v_intr_setstate,#function -sun4v_intr_setstate: - mov HV_FAST_INTR_SETSTATE, %o5 - ta HV_FAST_TRAP - retl - nop - .size sun4v_intr_setstate, .-sun4v_intr_setstate - - /* %o0: sysino - * - * returns %o0: cpuid - */ - .globl sun4v_intr_gettarget - .type sun4v_intr_gettarget,#function -sun4v_intr_gettarget: - mov HV_FAST_INTR_GETTARGET, %o5 - ta HV_FAST_TRAP - retl - mov %o1, %o0 - .size sun4v_intr_gettarget, .-sun4v_intr_gettarget - - /* %o0: sysino - * %o1: cpuid - */ - .globl sun4v_intr_settarget - .type sun4v_intr_settarget,#function -sun4v_intr_settarget: - mov HV_FAST_INTR_SETTARGET, %o5 - ta HV_FAST_TRAP - retl - nop - .size sun4v_intr_settarget, .-sun4v_intr_settarget - - /* %o0: cpuid - * %o1: pc - * %o2: rtba - * %o3: arg0 - * - * returns %o0: status - */ - .globl sun4v_cpu_start - .type sun4v_cpu_start,#function -sun4v_cpu_start: - mov HV_FAST_CPU_START, %o5 - ta HV_FAST_TRAP - retl - nop - .size sun4v_cpu_start, .-sun4v_cpu_start - - /* %o0: cpuid - * - * returns %o0: status - */ - .globl sun4v_cpu_stop - .type sun4v_cpu_stop,#function -sun4v_cpu_stop: - mov HV_FAST_CPU_STOP, %o5 - ta HV_FAST_TRAP - retl - nop - .size sun4v_cpu_stop, .-sun4v_cpu_stop - - /* returns %o0: status */ - .globl sun4v_cpu_yield - .type sun4v_cpu_yield, #function -sun4v_cpu_yield: - mov HV_FAST_CPU_YIELD, %o5 - ta HV_FAST_TRAP - retl - nop - .size sun4v_cpu_yield, .-sun4v_cpu_yield - - /* %o0: type - * %o1: queue paddr - * %o2: num queue entries - * - * returns %o0: status - */ - .globl sun4v_cpu_qconf - .type sun4v_cpu_qconf,#function -sun4v_cpu_qconf: - mov HV_FAST_CPU_QCONF, %o5 - ta HV_FAST_TRAP - retl - nop - .size sun4v_cpu_qconf, .-sun4v_cpu_qconf - - /* %o0: num cpus in cpu list - * %o1: cpu list paddr - * %o2: mondo block paddr - * - * returns %o0: status - */ - .globl sun4v_cpu_mondo_send - .type sun4v_cpu_mondo_send,#function -sun4v_cpu_mondo_send: - mov HV_FAST_CPU_MONDO_SEND, %o5 - ta HV_FAST_TRAP - retl - nop - .size sun4v_cpu_mondo_send, .-sun4v_cpu_mondo_send - - /* %o0: CPU ID - * - * returns %o0: -status if status non-zero, else - * %o0: cpu state as HV_CPU_STATE_* - */ - .globl sun4v_cpu_state - .type sun4v_cpu_state,#function -sun4v_cpu_state: - mov HV_FAST_CPU_STATE, %o5 - ta HV_FAST_TRAP - brnz,pn %o0, 1f - sub %g0, %o0, %o0 - mov %o1, %o0 -1: retl - nop - .size sun4v_cpu_state, .-sun4v_cpu_state - - /* %o0: virtual address - * %o1: must be zero - * %o2: TTE - * %o3: HV_MMU_* flags - * - * returns %o0: status - */ - .globl sun4v_mmu_map_perm_addr - .type sun4v_mmu_map_perm_addr,#function -sun4v_mmu_map_perm_addr: - mov HV_FAST_MMU_MAP_PERM_ADDR, %o5 - ta HV_FAST_TRAP - retl - nop - .size sun4v_mmu_map_perm_addr, .-sun4v_mmu_map_perm_addr - - /* %o0: number of TSB descriptions - * %o1: TSB descriptions real address - * - * returns %o0: status - */ - .globl sun4v_mmu_tsb_ctx0 - .type sun4v_mmu_tsb_ctx0,#function -sun4v_mmu_tsb_ctx0: - mov HV_FAST_MMU_TSB_CTX0, %o5 - ta HV_FAST_TRAP - retl - nop - .size sun4v_mmu_tsb_ctx0, .-sun4v_mmu_tsb_ctx0 - - /* %o0: API group number - * %o1: pointer to unsigned long major number storage - * %o2: pointer to unsigned long minor number storage - * - * returns %o0: status - */ - .globl sun4v_get_version - .type sun4v_get_version,#function -sun4v_get_version: - mov HV_CORE_GET_VER, %o5 - mov %o1, %o3 - mov %o2, %o4 - ta HV_CORE_TRAP - stx %o1, [%o3] - retl - stx %o2, [%o4] - .size sun4v_get_version, .-sun4v_get_version - - /* %o0: API group number - * %o1: desired major number - * %o2: desired minor number - * %o3: pointer to unsigned long actual minor number storage - * - * returns %o0: status - */ - .globl sun4v_set_version - .type sun4v_set_version,#function -sun4v_set_version: - mov HV_CORE_SET_VER, %o5 - mov %o3, %o4 - ta HV_CORE_TRAP - retl - stx %o1, [%o4] - .size sun4v_set_version, .-sun4v_set_version - - /* %o0: pointer to unsigned long time - * - * returns %o0: status - */ - .globl sun4v_tod_get - .type sun4v_tod_get,#function -sun4v_tod_get: - mov %o0, %o4 - mov HV_FAST_TOD_GET, %o5 - ta HV_FAST_TRAP - stx %o1, [%o4] - retl - nop - .size sun4v_tod_get, .-sun4v_tod_get - - /* %o0: time - * - * returns %o0: status - */ - .globl sun4v_tod_set - .type sun4v_tod_set,#function -sun4v_tod_set: - mov HV_FAST_TOD_SET, %o5 - ta HV_FAST_TRAP - retl - nop - .size sun4v_tod_set, .-sun4v_tod_set - - /* %o0: pointer to unsigned long status - * - * returns %o0: signed character - */ - .globl sun4v_con_getchar - .type sun4v_con_getchar,#function -sun4v_con_getchar: - mov %o0, %o4 - mov HV_FAST_CONS_GETCHAR, %o5 - clr %o0 - clr %o1 - ta HV_FAST_TRAP - stx %o0, [%o4] - retl - sra %o1, 0, %o0 - .size sun4v_con_getchar, .-sun4v_con_getchar - - /* %o0: signed long character - * - * returns %o0: status - */ - .globl sun4v_con_putchar - .type sun4v_con_putchar,#function -sun4v_con_putchar: - mov HV_FAST_CONS_PUTCHAR, %o5 - ta HV_FAST_TRAP - retl - sra %o0, 0, %o0 - .size sun4v_con_putchar, .-sun4v_con_putchar - - /* %o0: buffer real address - * %o1: buffer size - * %o2: pointer to unsigned long bytes_read - * - * returns %o0: status - */ - .globl sun4v_con_read - .type sun4v_con_read,#function -sun4v_con_read: - mov %o2, %o4 - mov HV_FAST_CONS_READ, %o5 - ta HV_FAST_TRAP - brnz %o0, 1f - cmp %o1, -1 /* break */ - be,a,pn %icc, 1f - mov %o1, %o0 - cmp %o1, -2 /* hup */ - be,a,pn %icc, 1f - mov %o1, %o0 - stx %o1, [%o4] -1: retl - nop - .size sun4v_con_read, .-sun4v_con_read - - /* %o0: buffer real address - * %o1: buffer size - * %o2: pointer to unsigned long bytes_written - * - * returns %o0: status - */ - .globl sun4v_con_write - .type sun4v_con_write,#function -sun4v_con_write: - mov %o2, %o4 - mov HV_FAST_CONS_WRITE, %o5 - ta HV_FAST_TRAP - stx %o1, [%o4] - retl - nop - .size sun4v_con_write, .-sun4v_con_write - - /* %o0: soft state - * %o1: address of description string - * - * returns %o0: status - */ - .globl sun4v_mach_set_soft_state - .type sun4v_mach_set_soft_state,#function -sun4v_mach_set_soft_state: - mov HV_FAST_MACH_SET_SOFT_STATE, %o5 - ta HV_FAST_TRAP - retl - nop - .size sun4v_mach_set_soft_state, .-sun4v_mach_set_soft_state - - /* %o0: exit code - * - * Does not return. - */ - .globl sun4v_mach_exit - .type sun4v_mach_exit,#function -sun4v_mach_exit: - mov HV_FAST_MACH_EXIT, %o5 - ta HV_FAST_TRAP - retl - nop - .size sun4v_mach_exit, .-sun4v_mach_exit - - /* %o0: buffer real address - * %o1: buffer length - * %o2: pointer to unsigned long real_buf_len - * - * returns %o0: status - */ - .globl sun4v_mach_desc - .type sun4v_mach_desc,#function -sun4v_mach_desc: - mov %o2, %o4 - mov HV_FAST_MACH_DESC, %o5 - ta HV_FAST_TRAP - stx %o1, [%o4] - retl - nop - .size sun4v_mach_desc, .-sun4v_mach_desc - - /* %o0: new timeout in milliseconds - * %o1: pointer to unsigned long orig_timeout - * - * returns %o0: status - */ - .globl sun4v_mach_set_watchdog - .type sun4v_mach_set_watchdog,#function -sun4v_mach_set_watchdog: - mov %o1, %o4 - mov HV_FAST_MACH_SET_WATCHDOG, %o5 - ta HV_FAST_TRAP - stx %o1, [%o4] - retl - nop - .size sun4v_mach_set_watchdog, .-sun4v_mach_set_watchdog - - /* No inputs and does not return. */ - .globl sun4v_mach_sir - .type sun4v_mach_sir,#function -sun4v_mach_sir: - mov %o1, %o4 - mov HV_FAST_MACH_SIR, %o5 - ta HV_FAST_TRAP - stx %o1, [%o4] - retl - nop - .size sun4v_mach_sir, .-sun4v_mach_sir - - /* %o0: channel - * %o1: ra - * %o2: num_entries - * - * returns %o0: status - */ - .globl sun4v_ldc_tx_qconf - .type sun4v_ldc_tx_qconf,#function -sun4v_ldc_tx_qconf: - mov HV_FAST_LDC_TX_QCONF, %o5 - ta HV_FAST_TRAP - retl - nop - .size sun4v_ldc_tx_qconf, .-sun4v_ldc_tx_qconf - - /* %o0: channel - * %o1: pointer to unsigned long ra - * %o2: pointer to unsigned long num_entries - * - * returns %o0: status - */ - .globl sun4v_ldc_tx_qinfo - .type sun4v_ldc_tx_qinfo,#function -sun4v_ldc_tx_qinfo: - mov %o1, %g1 - mov %o2, %g2 - mov HV_FAST_LDC_TX_QINFO, %o5 - ta HV_FAST_TRAP - stx %o1, [%g1] - stx %o2, [%g2] - retl - nop - .size sun4v_ldc_tx_qinfo, .-sun4v_ldc_tx_qinfo - - /* %o0: channel - * %o1: pointer to unsigned long head_off - * %o2: pointer to unsigned long tail_off - * %o2: pointer to unsigned long chan_state - * - * returns %o0: status - */ - .globl sun4v_ldc_tx_get_state - .type sun4v_ldc_tx_get_state,#function -sun4v_ldc_tx_get_state: - mov %o1, %g1 - mov %o2, %g2 - mov %o3, %g3 - mov HV_FAST_LDC_TX_GET_STATE, %o5 - ta HV_FAST_TRAP - stx %o1, [%g1] - stx %o2, [%g2] - stx %o3, [%g3] - retl - nop - .size sun4v_ldc_tx_get_state, .-sun4v_ldc_tx_get_state - - /* %o0: channel - * %o1: tail_off - * - * returns %o0: status - */ - .globl sun4v_ldc_tx_set_qtail - .type sun4v_ldc_tx_set_qtail,#function -sun4v_ldc_tx_set_qtail: - mov HV_FAST_LDC_TX_SET_QTAIL, %o5 - ta HV_FAST_TRAP - retl - nop - .size sun4v_ldc_tx_set_qtail, .-sun4v_ldc_tx_set_qtail - - /* %o0: channel - * %o1: ra - * %o2: num_entries - * - * returns %o0: status - */ - .globl sun4v_ldc_rx_qconf - .type sun4v_ldc_rx_qconf,#function -sun4v_ldc_rx_qconf: - mov HV_FAST_LDC_RX_QCONF, %o5 - ta HV_FAST_TRAP - retl - nop - .size sun4v_ldc_rx_qconf, .-sun4v_ldc_rx_qconf - - /* %o0: channel - * %o1: pointer to unsigned long ra - * %o2: pointer to unsigned long num_entries - * - * returns %o0: status - */ - .globl sun4v_ldc_rx_qinfo - .type sun4v_ldc_rx_qinfo,#function -sun4v_ldc_rx_qinfo: - mov %o1, %g1 - mov %o2, %g2 - mov HV_FAST_LDC_RX_QINFO, %o5 - ta HV_FAST_TRAP - stx %o1, [%g1] - stx %o2, [%g2] - retl - nop - .size sun4v_ldc_rx_qinfo, .-sun4v_ldc_rx_qinfo - - /* %o0: channel - * %o1: pointer to unsigned long head_off - * %o2: pointer to unsigned long tail_off - * %o2: pointer to unsigned long chan_state - * - * returns %o0: status - */ - .globl sun4v_ldc_rx_get_state - .type sun4v_ldc_rx_get_state,#function -sun4v_ldc_rx_get_state: - mov %o1, %g1 - mov %o2, %g2 - mov %o3, %g3 - mov HV_FAST_LDC_RX_GET_STATE, %o5 - ta HV_FAST_TRAP - stx %o1, [%g1] - stx %o2, [%g2] - stx %o3, [%g3] - retl - nop - .size sun4v_ldc_rx_get_state, .-sun4v_ldc_rx_get_state - - /* %o0: channel - * %o1: head_off - * - * returns %o0: status - */ - .globl sun4v_ldc_rx_set_qhead - .type sun4v_ldc_rx_set_qhead,#function -sun4v_ldc_rx_set_qhead: - mov HV_FAST_LDC_RX_SET_QHEAD, %o5 - ta HV_FAST_TRAP - retl - nop - .size sun4v_ldc_rx_set_qhead, .-sun4v_ldc_rx_set_qhead - - /* %o0: channel - * %o1: ra - * %o2: num_entries - * - * returns %o0: status - */ - .globl sun4v_ldc_set_map_table - .type sun4v_ldc_set_map_table,#function -sun4v_ldc_set_map_table: - mov HV_FAST_LDC_SET_MAP_TABLE, %o5 - ta HV_FAST_TRAP - retl - nop - .size sun4v_ldc_set_map_table, .-sun4v_ldc_set_map_table - - /* %o0: channel - * %o1: pointer to unsigned long ra - * %o2: pointer to unsigned long num_entries - * - * returns %o0: status - */ - .globl sun4v_ldc_get_map_table - .type sun4v_ldc_get_map_table,#function -sun4v_ldc_get_map_table: - mov %o1, %g1 - mov %o2, %g2 - mov HV_FAST_LDC_GET_MAP_TABLE, %o5 - ta HV_FAST_TRAP - stx %o1, [%g1] - stx %o2, [%g2] - retl - nop - .size sun4v_ldc_get_map_table, .-sun4v_ldc_get_map_table - - /* %o0: channel - * %o1: dir_code - * %o2: tgt_raddr - * %o3: lcl_raddr - * %o4: len - * %o5: pointer to unsigned long actual_len - * - * returns %o0: status - */ - .globl sun4v_ldc_copy - .type sun4v_ldc_copy,#function -sun4v_ldc_copy: - mov %o5, %g1 - mov HV_FAST_LDC_COPY, %o5 - ta HV_FAST_TRAP - stx %o1, [%g1] - retl - nop - .size sun4v_ldc_copy, .-sun4v_ldc_copy - - /* %o0: channel - * %o1: cookie - * %o2: pointer to unsigned long ra - * %o3: pointer to unsigned long perm - * - * returns %o0: status - */ - .globl sun4v_ldc_mapin - .type sun4v_ldc_mapin,#function -sun4v_ldc_mapin: - mov %o2, %g1 - mov %o3, %g2 - mov HV_FAST_LDC_MAPIN, %o5 - ta HV_FAST_TRAP - stx %o1, [%g1] - stx %o2, [%g2] - retl - nop - .size sun4v_ldc_mapin, .-sun4v_ldc_mapin - - /* %o0: ra - * - * returns %o0: status - */ - .globl sun4v_ldc_unmap - .type sun4v_ldc_unmap,#function -sun4v_ldc_unmap: - mov HV_FAST_LDC_UNMAP, %o5 - ta HV_FAST_TRAP - retl - nop - .size sun4v_ldc_unmap, .-sun4v_ldc_unmap - - /* %o0: channel - * %o1: cookie - * %o2: mte_cookie - * - * returns %o0: status - */ - .globl sun4v_ldc_revoke - .type sun4v_ldc_revoke,#function -sun4v_ldc_revoke: - mov HV_FAST_LDC_REVOKE, %o5 - ta HV_FAST_TRAP - retl - nop - .size sun4v_ldc_revoke, .-sun4v_ldc_revoke - - /* %o0: device handle - * %o1: device INO - * %o2: pointer to unsigned long cookie - * - * returns %o0: status - */ - .globl sun4v_vintr_get_cookie - .type sun4v_vintr_get_cookie,#function -sun4v_vintr_get_cookie: - mov %o2, %g1 - mov HV_FAST_VINTR_GET_COOKIE, %o5 - ta HV_FAST_TRAP - stx %o1, [%g1] - retl - nop - .size sun4v_vintr_get_cookie, .-sun4v_vintr_get_cookie - - /* %o0: device handle - * %o1: device INO - * %o2: cookie - * - * returns %o0: status - */ - .globl sun4v_vintr_set_cookie - .type sun4v_vintr_set_cookie,#function -sun4v_vintr_set_cookie: - mov HV_FAST_VINTR_SET_COOKIE, %o5 - ta HV_FAST_TRAP - retl - nop - .size sun4v_vintr_set_cookie, .-sun4v_vintr_set_cookie - - /* %o0: device handle - * %o1: device INO - * %o2: pointer to unsigned long valid_state - * - * returns %o0: status - */ - .globl sun4v_vintr_get_valid - .type sun4v_vintr_get_valid,#function -sun4v_vintr_get_valid: - mov %o2, %g1 - mov HV_FAST_VINTR_GET_VALID, %o5 - ta HV_FAST_TRAP - stx %o1, [%g1] - retl - nop - .size sun4v_vintr_get_valid, .-sun4v_vintr_get_valid - - /* %o0: device handle - * %o1: device INO - * %o2: valid_state - * - * returns %o0: status - */ - .globl sun4v_vintr_set_valid - .type sun4v_vintr_set_valid,#function -sun4v_vintr_set_valid: - mov HV_FAST_VINTR_SET_VALID, %o5 - ta HV_FAST_TRAP - retl - nop - .size sun4v_vintr_set_valid, .-sun4v_vintr_set_valid - - /* %o0: device handle - * %o1: device INO - * %o2: pointer to unsigned long state - * - * returns %o0: status - */ - .globl sun4v_vintr_get_state - .type sun4v_vintr_get_state,#function -sun4v_vintr_get_state: - mov %o2, %g1 - mov HV_FAST_VINTR_GET_STATE, %o5 - ta HV_FAST_TRAP - stx %o1, [%g1] - retl - nop - .size sun4v_vintr_get_state, .-sun4v_vintr_get_state - - /* %o0: device handle - * %o1: device INO - * %o2: state - * - * returns %o0: status - */ - .globl sun4v_vintr_set_state - .type sun4v_vintr_set_state,#function -sun4v_vintr_set_state: - mov HV_FAST_VINTR_SET_STATE, %o5 - ta HV_FAST_TRAP - retl - nop - .size sun4v_vintr_set_state, .-sun4v_vintr_set_state - - /* %o0: device handle - * %o1: device INO - * %o2: pointer to unsigned long cpuid - * - * returns %o0: status - */ - .globl sun4v_vintr_get_target - .type sun4v_vintr_get_target,#function -sun4v_vintr_get_target: - mov %o2, %g1 - mov HV_FAST_VINTR_GET_TARGET, %o5 - ta HV_FAST_TRAP - stx %o1, [%g1] - retl - nop - .size sun4v_vintr_get_target, .-sun4v_vintr_get_target - - /* %o0: device handle - * %o1: device INO - * %o2: cpuid - * - * returns %o0: status - */ - .globl sun4v_vintr_set_target - .type sun4v_vintr_set_target,#function -sun4v_vintr_set_target: - mov HV_FAST_VINTR_SET_TARGET, %o5 - ta HV_FAST_TRAP - retl - nop - .size sun4v_vintr_set_target, .-sun4v_vintr_set_target - - /* %o0: NCS sub-function - * %o1: sub-function arg real-address - * %o2: sub-function arg size - * - * returns %o0: status - */ - .globl sun4v_ncs_request - .type sun4v_ncs_request,#function -sun4v_ncs_request: - mov HV_FAST_NCS_REQUEST, %o5 - ta HV_FAST_TRAP - retl - nop - .size sun4v_ncs_request, .-sun4v_ncs_request - - .globl sun4v_svc_send - .type sun4v_svc_send,#function -sun4v_svc_send: - save %sp, -192, %sp - mov %i0, %o0 - mov %i1, %o1 - mov %i2, %o2 - mov HV_FAST_SVC_SEND, %o5 - ta HV_FAST_TRAP - stx %o1, [%i3] - ret - restore - .size sun4v_svc_send, .-sun4v_svc_send - - .globl sun4v_svc_recv - .type sun4v_svc_recv,#function -sun4v_svc_recv: - save %sp, -192, %sp - mov %i0, %o0 - mov %i1, %o1 - mov %i2, %o2 - mov HV_FAST_SVC_RECV, %o5 - ta HV_FAST_TRAP - stx %o1, [%i3] - ret - restore - .size sun4v_svc_recv, .-sun4v_svc_recv - - .globl sun4v_svc_getstatus - .type sun4v_svc_getstatus,#function -sun4v_svc_getstatus: - mov HV_FAST_SVC_GETSTATUS, %o5 - mov %o1, %o4 - ta HV_FAST_TRAP - stx %o1, [%o4] - retl - nop - .size sun4v_svc_getstatus, .-sun4v_svc_getstatus - - .globl sun4v_svc_setstatus - .type sun4v_svc_setstatus,#function -sun4v_svc_setstatus: - mov HV_FAST_SVC_SETSTATUS, %o5 - ta HV_FAST_TRAP - retl - nop - .size sun4v_svc_setstatus, .-sun4v_svc_setstatus - - .globl sun4v_svc_clrstatus - .type sun4v_svc_clrstatus,#function -sun4v_svc_clrstatus: - mov HV_FAST_SVC_CLRSTATUS, %o5 - ta HV_FAST_TRAP - retl - nop - .size sun4v_svc_clrstatus, .-sun4v_svc_clrstatus - - .globl sun4v_mmustat_conf - .type sun4v_mmustat_conf,#function -sun4v_mmustat_conf: - mov %o1, %o4 - mov HV_FAST_MMUSTAT_CONF, %o5 - ta HV_FAST_TRAP - stx %o1, [%o4] - retl - nop - .size sun4v_mmustat_conf, .-sun4v_mmustat_conf - - .globl sun4v_mmustat_info - .type sun4v_mmustat_info,#function -sun4v_mmustat_info: - mov %o0, %o4 - mov HV_FAST_MMUSTAT_INFO, %o5 - ta HV_FAST_TRAP - stx %o1, [%o4] - retl - nop - .size sun4v_mmustat_info, .-sun4v_mmustat_info - - .globl sun4v_mmu_demap_all - .type sun4v_mmu_demap_all,#function -sun4v_mmu_demap_all: - clr %o0 - clr %o1 - mov HV_MMU_ALL, %o2 - mov HV_FAST_MMU_DEMAP_ALL, %o5 - ta HV_FAST_TRAP - retl - nop - .size sun4v_mmu_demap_all, .-sun4v_mmu_demap_all diff --git a/arch/sparc64/kernel/entry.h b/arch/sparc64/kernel/entry.h index 4a91e9c6d31..32fbab62085 100644 --- a/arch/sparc64/kernel/entry.h +++ b/arch/sparc64/kernel/entry.h @@ -20,7 +20,6 @@ extern void timer_interrupt(int irq, struct pt_regs *regs); extern void do_notify_resume(struct pt_regs *regs, unsigned long orig_i0, - int restart_syscall, unsigned long thread_info_flags); extern asmlinkage void syscall_trace(struct pt_regs *regs, diff --git a/arch/sparc64/kernel/etrap.S b/arch/sparc64/kernel/etrap.S index 4b2bf9eb447..b49d3b60bc0 100644 --- a/arch/sparc64/kernel/etrap.S +++ b/arch/sparc64/kernel/etrap.S @@ -53,7 +53,11 @@ etrap_irq: stx %g3, [%g2 + STACKFRAME_SZ + PT_V9_TPC] rd %y, %g3 stx %g1, [%g2 + STACKFRAME_SZ + PT_V9_TNPC] + rdpr %tt, %g1 st %g3, [%g2 + STACKFRAME_SZ + PT_V9_Y] + sethi %hi(PT_REGS_MAGIC), %g3 + or %g3, %g1, %g1 + st %g1, [%g2 + STACKFRAME_SZ + PT_V9_MAGIC] rdpr %cansave, %g1 brnz,pt %g1, etrap_save diff --git a/arch/sparc64/kernel/fpu_traps.S b/arch/sparc64/kernel/fpu_traps.S new file mode 100644 index 00000000000..a6864826a4b --- /dev/null +++ b/arch/sparc64/kernel/fpu_traps.S @@ -0,0 +1,384 @@ + /* This is trivial with the new code... */ + .globl do_fpdis + .type do_fpdis,#function +do_fpdis: + sethi %hi(TSTATE_PEF), %g4 + rdpr %tstate, %g5 + andcc %g5, %g4, %g0 + be,pt %xcc, 1f + nop + rd %fprs, %g5 + andcc %g5, FPRS_FEF, %g0 + be,pt %xcc, 1f + nop + + /* Legal state when DCR_IFPOE is set in Cheetah %dcr. */ + sethi %hi(109f), %g7 + ba,pt %xcc, etrap +109: or %g7, %lo(109b), %g7 + add %g0, %g0, %g0 + ba,a,pt %xcc, rtrap + +1: TRAP_LOAD_THREAD_REG(%g6, %g1) + ldub [%g6 + TI_FPSAVED], %g5 + wr %g0, FPRS_FEF, %fprs + andcc %g5, FPRS_FEF, %g0 + be,a,pt %icc, 1f + clr %g7 + ldx [%g6 + TI_GSR], %g7 +1: andcc %g5, FPRS_DL, %g0 + bne,pn %icc, 2f + fzero %f0 + andcc %g5, FPRS_DU, %g0 + bne,pn %icc, 1f + fzero %f2 + faddd %f0, %f2, %f4 + fmuld %f0, %f2, %f6 + faddd %f0, %f2, %f8 + fmuld %f0, %f2, %f10 + faddd %f0, %f2, %f12 + fmuld %f0, %f2, %f14 + faddd %f0, %f2, %f16 + fmuld %f0, %f2, %f18 + faddd %f0, %f2, %f20 + fmuld %f0, %f2, %f22 + faddd %f0, %f2, %f24 + fmuld %f0, %f2, %f26 + faddd %f0, %f2, %f28 + fmuld %f0, %f2, %f30 + faddd %f0, %f2, %f32 + fmuld %f0, %f2, %f34 + faddd %f0, %f2, %f36 + fmuld %f0, %f2, %f38 + faddd %f0, %f2, %f40 + fmuld %f0, %f2, %f42 + faddd %f0, %f2, %f44 + fmuld %f0, %f2, %f46 + faddd %f0, %f2, %f48 + fmuld %f0, %f2, %f50 + faddd %f0, %f2, %f52 + fmuld %f0, %f2, %f54 + faddd %f0, %f2, %f56 + fmuld %f0, %f2, %f58 + b,pt %xcc, fpdis_exit2 + faddd %f0, %f2, %f60 +1: mov SECONDARY_CONTEXT, %g3 + add %g6, TI_FPREGS + 0x80, %g1 + faddd %f0, %f2, %f4 + fmuld %f0, %f2, %f6 + +661: ldxa [%g3] ASI_DMMU, %g5 + .section .sun4v_1insn_patch, "ax" + .word 661b + ldxa [%g3] ASI_MMU, %g5 + .previous + + sethi %hi(sparc64_kern_sec_context), %g2 + ldx [%g2 + %lo(sparc64_kern_sec_context)], %g2 + +661: stxa %g2, [%g3] ASI_DMMU + .section .sun4v_1insn_patch, "ax" + .word 661b + stxa %g2, [%g3] ASI_MMU + .previous + + membar #Sync + add %g6, TI_FPREGS + 0xc0, %g2 + faddd %f0, %f2, %f8 + fmuld %f0, %f2, %f10 + membar #Sync + ldda [%g1] ASI_BLK_S, %f32 + ldda [%g2] ASI_BLK_S, %f48 + membar #Sync + faddd %f0, %f2, %f12 + fmuld %f0, %f2, %f14 + faddd %f0, %f2, %f16 + fmuld %f0, %f2, %f18 + faddd %f0, %f2, %f20 + fmuld %f0, %f2, %f22 + faddd %f0, %f2, %f24 + fmuld %f0, %f2, %f26 + faddd %f0, %f2, %f28 + fmuld %f0, %f2, %f30 + b,pt %xcc, fpdis_exit + nop +2: andcc %g5, FPRS_DU, %g0 + bne,pt %icc, 3f + fzero %f32 + mov SECONDARY_CONTEXT, %g3 + fzero %f34 + +661: ldxa [%g3] ASI_DMMU, %g5 + .section .sun4v_1insn_patch, "ax" + .word 661b + ldxa [%g3] ASI_MMU, %g5 + .previous + + add %g6, TI_FPREGS, %g1 + sethi %hi(sparc64_kern_sec_context), %g2 + ldx [%g2 + %lo(sparc64_kern_sec_context)], %g2 + +661: stxa %g2, [%g3] ASI_DMMU + .section .sun4v_1insn_patch, "ax" + .word 661b + stxa %g2, [%g3] ASI_MMU + .previous + + membar #Sync + add %g6, TI_FPREGS + 0x40, %g2 + faddd %f32, %f34, %f36 + fmuld %f32, %f34, %f38 + membar #Sync + ldda [%g1] ASI_BLK_S, %f0 + ldda [%g2] ASI_BLK_S, %f16 + membar #Sync + faddd %f32, %f34, %f40 + fmuld %f32, %f34, %f42 + faddd %f32, %f34, %f44 + fmuld %f32, %f34, %f46 + faddd %f32, %f34, %f48 + fmuld %f32, %f34, %f50 + faddd %f32, %f34, %f52 + fmuld %f32, %f34, %f54 + faddd %f32, %f34, %f56 + fmuld %f32, %f34, %f58 + faddd %f32, %f34, %f60 + fmuld %f32, %f34, %f62 + ba,pt %xcc, fpdis_exit + nop +3: mov SECONDARY_CONTEXT, %g3 + add %g6, TI_FPREGS, %g1 + +661: ldxa [%g3] ASI_DMMU, %g5 + .section .sun4v_1insn_patch, "ax" + .word 661b + ldxa [%g3] ASI_MMU, %g5 + .previous + + sethi %hi(sparc64_kern_sec_context), %g2 + ldx [%g2 + %lo(sparc64_kern_sec_context)], %g2 + +661: stxa %g2, [%g3] ASI_DMMU + .section .sun4v_1insn_patch, "ax" + .word 661b + stxa %g2, [%g3] ASI_MMU + .previous + + membar #Sync + mov 0x40, %g2 + membar #Sync + ldda [%g1] ASI_BLK_S, %f0 + ldda [%g1 + %g2] ASI_BLK_S, %f16 + add %g1, 0x80, %g1 + ldda [%g1] ASI_BLK_S, %f32 + ldda [%g1 + %g2] ASI_BLK_S, %f48 + membar #Sync +fpdis_exit: + +661: stxa %g5, [%g3] ASI_DMMU + .section .sun4v_1insn_patch, "ax" + .word 661b + stxa %g5, [%g3] ASI_MMU + .previous + + membar #Sync +fpdis_exit2: + wr %g7, 0, %gsr + ldx [%g6 + TI_XFSR], %fsr + rdpr %tstate, %g3 + or %g3, %g4, %g3 ! anal... + wrpr %g3, %tstate + wr %g0, FPRS_FEF, %fprs ! clean DU/DL bits + retry + .size do_fpdis,.-do_fpdis + + .align 32 + .type fp_other_bounce,#function +fp_other_bounce: + call do_fpother + add %sp, PTREGS_OFF, %o0 + ba,pt %xcc, rtrap + nop + .size fp_other_bounce,.-fp_other_bounce + + .align 32 + .globl do_fpother_check_fitos + .type do_fpother_check_fitos,#function +do_fpother_check_fitos: + TRAP_LOAD_THREAD_REG(%g6, %g1) + sethi %hi(fp_other_bounce - 4), %g7 + or %g7, %lo(fp_other_bounce - 4), %g7 + + /* NOTE: Need to preserve %g7 until we fully commit + * to the fitos fixup. + */ + stx %fsr, [%g6 + TI_XFSR] + rdpr %tstate, %g3 + andcc %g3, TSTATE_PRIV, %g0 + bne,pn %xcc, do_fptrap_after_fsr + nop + ldx [%g6 + TI_XFSR], %g3 + srlx %g3, 14, %g1 + and %g1, 7, %g1 + cmp %g1, 2 ! Unfinished FP-OP + bne,pn %xcc, do_fptrap_after_fsr + sethi %hi(1 << 23), %g1 ! Inexact + andcc %g3, %g1, %g0 + bne,pn %xcc, do_fptrap_after_fsr + rdpr %tpc, %g1 + lduwa [%g1] ASI_AIUP, %g3 ! This cannot ever fail +#define FITOS_MASK 0xc1f83fe0 +#define FITOS_COMPARE 0x81a01880 + sethi %hi(FITOS_MASK), %g1 + or %g1, %lo(FITOS_MASK), %g1 + and %g3, %g1, %g1 + sethi %hi(FITOS_COMPARE), %g2 + or %g2, %lo(FITOS_COMPARE), %g2 + cmp %g1, %g2 + bne,pn %xcc, do_fptrap_after_fsr + nop + std %f62, [%g6 + TI_FPREGS + (62 * 4)] + sethi %hi(fitos_table_1), %g1 + and %g3, 0x1f, %g2 + or %g1, %lo(fitos_table_1), %g1 + sllx %g2, 2, %g2 + jmpl %g1 + %g2, %g0 + ba,pt %xcc, fitos_emul_continue + +fitos_table_1: + fitod %f0, %f62 + fitod %f1, %f62 + fitod %f2, %f62 + fitod %f3, %f62 + fitod %f4, %f62 + fitod %f5, %f62 + fitod %f6, %f62 + fitod %f7, %f62 + fitod %f8, %f62 + fitod %f9, %f62 + fitod %f10, %f62 + fitod %f11, %f62 + fitod %f12, %f62 + fitod %f13, %f62 + fitod %f14, %f62 + fitod %f15, %f62 + fitod %f16, %f62 + fitod %f17, %f62 + fitod %f18, %f62 + fitod %f19, %f62 + fitod %f20, %f62 + fitod %f21, %f62 + fitod %f22, %f62 + fitod %f23, %f62 + fitod %f24, %f62 + fitod %f25, %f62 + fitod %f26, %f62 + fitod %f27, %f62 + fitod %f28, %f62 + fitod %f29, %f62 + fitod %f30, %f62 + fitod %f31, %f62 + +fitos_emul_continue: + sethi %hi(fitos_table_2), %g1 + srl %g3, 25, %g2 + or %g1, %lo(fitos_table_2), %g1 + and %g2, 0x1f, %g2 + sllx %g2, 2, %g2 + jmpl %g1 + %g2, %g0 + ba,pt %xcc, fitos_emul_fini + +fitos_table_2: + fdtos %f62, %f0 + fdtos %f62, %f1 + fdtos %f62, %f2 + fdtos %f62, %f3 + fdtos %f62, %f4 + fdtos %f62, %f5 + fdtos %f62, %f6 + fdtos %f62, %f7 + fdtos %f62, %f8 + fdtos %f62, %f9 + fdtos %f62, %f10 + fdtos %f62, %f11 + fdtos %f62, %f12 + fdtos %f62, %f13 + fdtos %f62, %f14 + fdtos %f62, %f15 + fdtos %f62, %f16 + fdtos %f62, %f17 + fdtos %f62, %f18 + fdtos %f62, %f19 + fdtos %f62, %f20 + fdtos %f62, %f21 + fdtos %f62, %f22 + fdtos %f62, %f23 + fdtos %f62, %f24 + fdtos %f62, %f25 + fdtos %f62, %f26 + fdtos %f62, %f27 + fdtos %f62, %f28 + fdtos %f62, %f29 + fdtos %f62, %f30 + fdtos %f62, %f31 + +fitos_emul_fini: + ldd [%g6 + TI_FPREGS + (62 * 4)], %f62 + done + .size do_fpother_check_fitos,.-do_fpother_check_fitos + + .align 32 + .globl do_fptrap + .type do_fptrap,#function +do_fptrap: + TRAP_LOAD_THREAD_REG(%g6, %g1) + stx %fsr, [%g6 + TI_XFSR] +do_fptrap_after_fsr: + ldub [%g6 + TI_FPSAVED], %g3 + rd %fprs, %g1 + or %g3, %g1, %g3 + stb %g3, [%g6 + TI_FPSAVED] + rd %gsr, %g3 + stx %g3, [%g6 + TI_GSR] + mov SECONDARY_CONTEXT, %g3 + +661: ldxa [%g3] ASI_DMMU, %g5 + .section .sun4v_1insn_patch, "ax" + .word 661b + ldxa [%g3] ASI_MMU, %g5 + .previous + + sethi %hi(sparc64_kern_sec_context), %g2 + ldx [%g2 + %lo(sparc64_kern_sec_context)], %g2 + +661: stxa %g2, [%g3] ASI_DMMU + .section .sun4v_1insn_patch, "ax" + .word 661b + stxa %g2, [%g3] ASI_MMU + .previous + + membar #Sync + add %g6, TI_FPREGS, %g2 + andcc %g1, FPRS_DL, %g0 + be,pn %icc, 4f + mov 0x40, %g3 + stda %f0, [%g2] ASI_BLK_S + stda %f16, [%g2 + %g3] ASI_BLK_S + andcc %g1, FPRS_DU, %g0 + be,pn %icc, 5f +4: add %g2, 128, %g2 + stda %f32, [%g2] ASI_BLK_S + stda %f48, [%g2 + %g3] ASI_BLK_S +5: mov SECONDARY_CONTEXT, %g1 + membar #Sync + +661: stxa %g5, [%g1] ASI_DMMU + .section .sun4v_1insn_patch, "ax" + .word 661b + stxa %g5, [%g1] ASI_MMU + .previous + + membar #Sync + ba,pt %xcc, etrap + wr %g0, 0, %fprs + .size do_fptrap,.-do_fptrap diff --git a/arch/sparc64/kernel/getsetcc.S b/arch/sparc64/kernel/getsetcc.S new file mode 100644 index 00000000000..a14d272d206 --- /dev/null +++ b/arch/sparc64/kernel/getsetcc.S @@ -0,0 +1,24 @@ + .globl getcc + .type getcc,#function +getcc: + ldx [%o0 + PT_V9_TSTATE], %o1 + srlx %o1, 32, %o1 + and %o1, 0xf, %o1 + retl + stx %o1, [%o0 + PT_V9_G1] + .size getcc,.-getcc + + .globl setcc + .type setcc,#function +setcc: + ldx [%o0 + PT_V9_TSTATE], %o1 + ldx [%o0 + PT_V9_G1], %o2 + or %g0, %ulo(TSTATE_ICC), %o3 + sllx %o3, 32, %o3 + andn %o1, %o3, %o1 + sllx %o2, 32, %o2 + and %o2, %o3, %o2 + or %o1, %o2, %o1 + retl + stx %o1, [%o0 + PT_V9_TSTATE] + .size setcc,.-setcc diff --git a/arch/sparc64/kernel/head.S b/arch/sparc64/kernel/head.S index 34f8ff57c56..c9afef093d5 100644 --- a/arch/sparc64/kernel/head.S +++ b/arch/sparc64/kernel/head.S @@ -27,6 +27,10 @@ #include <asm/ttable.h> #include <asm/mmu.h> #include <asm/cpudata.h> +#include <asm/pil.h> +#include <asm/estate.h> +#include <asm/sfafsr.h> +#include <asm/unistd.h> /* This section from from _start to sparc64_boot_end should fit into * 0x0000000000404000 to 0x0000000000408000. @@ -823,7 +827,16 @@ sparc64_boot_end: #include "etrap.S" #include "rtrap.S" #include "winfixup.S" -#include "entry.S" +#include "fpu_traps.S" +#include "ivec.S" +#include "getsetcc.S" +#include "utrap.S" +#include "spiterrs.S" +#include "cherrs.S" +#include "misctrap.S" +#include "syscalls.S" +#include "helpers.S" +#include "hvcalls.S" #include "sun4v_tlb_miss.S" #include "sun4v_ivec.S" #include "ktlb.S" diff --git a/arch/sparc64/kernel/helpers.S b/arch/sparc64/kernel/helpers.S new file mode 100644 index 00000000000..314dd0c9fc5 --- /dev/null +++ b/arch/sparc64/kernel/helpers.S @@ -0,0 +1,63 @@ + .align 32 + .globl __flushw_user + .type __flushw_user,#function +__flushw_user: + rdpr %otherwin, %g1 + brz,pn %g1, 2f + clr %g2 +1: save %sp, -128, %sp + rdpr %otherwin, %g1 + brnz,pt %g1, 1b + add %g2, 1, %g2 +1: sub %g2, 1, %g2 + brnz,pt %g2, 1b + restore %g0, %g0, %g0 +2: retl + nop + .size __flushw_user,.-__flushw_user + + /* Flush %fp and %i7 to the stack for all register + * windows active inside of the cpu. This allows + * show_stack_trace() to avoid using an expensive + * 'flushw'. + */ + .globl stack_trace_flush + .type stack_trace_flush,#function +stack_trace_flush: + rdpr %pstate, %o0 + wrpr %o0, PSTATE_IE, %pstate + + rdpr %cwp, %g1 + rdpr %canrestore, %g2 + sub %g1, 1, %g3 + +1: brz,pn %g2, 2f + sub %g2, 1, %g2 + wrpr %g3, %cwp + stx %fp, [%sp + STACK_BIAS + RW_V9_I6] + stx %i7, [%sp + STACK_BIAS + RW_V9_I7] + ba,pt %xcc, 1b + sub %g3, 1, %g3 + +2: wrpr %g1, %cwp + wrpr %o0, %pstate + + retl + nop + .size stack_trace_flush,.-stack_trace_flush + +#ifdef CONFIG_SMP + .globl hard_smp_processor_id + .type hard_smp_processor_id,#function +hard_smp_processor_id: +#endif + .globl real_hard_smp_processor_id + .type real_hard_smp_processor_id,#function +real_hard_smp_processor_id: + __GET_CPUID(%o0) + retl + nop +#ifdef CONFIG_SMP + .size hard_smp_processor_id,.-hard_smp_processor_id +#endif + .size real_hard_smp_processor_id,.-real_hard_smp_processor_id diff --git a/arch/sparc64/kernel/hvcalls.S b/arch/sparc64/kernel/hvcalls.S new file mode 100644 index 00000000000..a2810f3ac70 --- /dev/null +++ b/arch/sparc64/kernel/hvcalls.S @@ -0,0 +1,886 @@ + /* %o0: devhandle + * %o1: devino + * + * returns %o0: sysino + */ + .globl sun4v_devino_to_sysino + .type sun4v_devino_to_sysino,#function +sun4v_devino_to_sysino: + mov HV_FAST_INTR_DEVINO2SYSINO, %o5 + ta HV_FAST_TRAP + retl + mov %o1, %o0 + .size sun4v_devino_to_sysino, .-sun4v_devino_to_sysino + + /* %o0: sysino + * + * returns %o0: intr_enabled (HV_INTR_{DISABLED,ENABLED}) + */ + .globl sun4v_intr_getenabled + .type sun4v_intr_getenabled,#function +sun4v_intr_getenabled: + mov HV_FAST_INTR_GETENABLED, %o5 + ta HV_FAST_TRAP + retl + mov %o1, %o0 + .size sun4v_intr_getenabled, .-sun4v_intr_getenabled + + /* %o0: sysino + * %o1: intr_enabled (HV_INTR_{DISABLED,ENABLED}) + */ + .globl sun4v_intr_setenabled + .type sun4v_intr_setenabled,#function +sun4v_intr_setenabled: + mov HV_FAST_INTR_SETENABLED, %o5 + ta HV_FAST_TRAP + retl + nop + .size sun4v_intr_setenabled, .-sun4v_intr_setenabled + + /* %o0: sysino + * + * returns %o0: intr_state (HV_INTR_STATE_*) + */ + .globl sun4v_intr_getstate + .type sun4v_intr_getstate,#function +sun4v_intr_getstate: + mov HV_FAST_INTR_GETSTATE, %o5 + ta HV_FAST_TRAP + retl + mov %o1, %o0 + .size sun4v_intr_getstate, .-sun4v_intr_getstate + + /* %o0: sysino + * %o1: intr_state (HV_INTR_STATE_*) + */ + .globl sun4v_intr_setstate + .type sun4v_intr_setstate,#function +sun4v_intr_setstate: + mov HV_FAST_INTR_SETSTATE, %o5 + ta HV_FAST_TRAP + retl + nop + .size sun4v_intr_setstate, .-sun4v_intr_setstate + + /* %o0: sysino + * + * returns %o0: cpuid + */ + .globl sun4v_intr_gettarget + .type sun4v_intr_gettarget,#function +sun4v_intr_gettarget: + mov HV_FAST_INTR_GETTARGET, %o5 + ta HV_FAST_TRAP + retl + mov %o1, %o0 + .size sun4v_intr_gettarget, .-sun4v_intr_gettarget + + /* %o0: sysino + * %o1: cpuid + */ + .globl sun4v_intr_settarget + .type sun4v_intr_settarget,#function +sun4v_intr_settarget: + mov HV_FAST_INTR_SETTARGET, %o5 + ta HV_FAST_TRAP + retl + nop + .size sun4v_intr_settarget, .-sun4v_intr_settarget + + /* %o0: cpuid + * %o1: pc + * %o2: rtba + * %o3: arg0 + * + * returns %o0: status + */ + .globl sun4v_cpu_start + .type sun4v_cpu_start,#function +sun4v_cpu_start: + mov HV_FAST_CPU_START, %o5 + ta HV_FAST_TRAP + retl + nop + .size sun4v_cpu_start, .-sun4v_cpu_start + + /* %o0: cpuid + * + * returns %o0: status + */ + .globl sun4v_cpu_stop + .type sun4v_cpu_stop,#function +sun4v_cpu_stop: + mov HV_FAST_CPU_STOP, %o5 + ta HV_FAST_TRAP + retl + nop + .size sun4v_cpu_stop, .-sun4v_cpu_stop + + /* returns %o0: status */ + .globl sun4v_cpu_yield + .type sun4v_cpu_yield, #function +sun4v_cpu_yield: + mov HV_FAST_CPU_YIELD, %o5 + ta HV_FAST_TRAP + retl + nop + .size sun4v_cpu_yield, .-sun4v_cpu_yield + + /* %o0: type + * %o1: queue paddr + * %o2: num queue entries + * + * returns %o0: status + */ + .globl sun4v_cpu_qconf + .type sun4v_cpu_qconf,#function +sun4v_cpu_qconf: + mov HV_FAST_CPU_QCONF, %o5 + ta HV_FAST_TRAP + retl + nop + .size sun4v_cpu_qconf, .-sun4v_cpu_qconf + + /* %o0: num cpus in cpu list + * %o1: cpu list paddr + * %o2: mondo block paddr + * + * returns %o0: status + */ + .globl sun4v_cpu_mondo_send + .type sun4v_cpu_mondo_send,#function +sun4v_cpu_mondo_send: + mov HV_FAST_CPU_MONDO_SEND, %o5 + ta HV_FAST_TRAP + retl + nop + .size sun4v_cpu_mondo_send, .-sun4v_cpu_mondo_send + + /* %o0: CPU ID + * + * returns %o0: -status if status non-zero, else + * %o0: cpu state as HV_CPU_STATE_* + */ + .globl sun4v_cpu_state + .type sun4v_cpu_state,#function +sun4v_cpu_state: + mov HV_FAST_CPU_STATE, %o5 + ta HV_FAST_TRAP + brnz,pn %o0, 1f + sub %g0, %o0, %o0 + mov %o1, %o0 +1: retl + nop + .size sun4v_cpu_state, .-sun4v_cpu_state + + /* %o0: virtual address + * %o1: must be zero + * %o2: TTE + * %o3: HV_MMU_* flags + * + * returns %o0: status + */ + .globl sun4v_mmu_map_perm_addr + .type sun4v_mmu_map_perm_addr,#function +sun4v_mmu_map_perm_addr: + mov HV_FAST_MMU_MAP_PERM_ADDR, %o5 + ta HV_FAST_TRAP + retl + nop + .size sun4v_mmu_map_perm_addr, .-sun4v_mmu_map_perm_addr + + /* %o0: number of TSB descriptions + * %o1: TSB descriptions real address + * + * returns %o0: status + */ + .globl sun4v_mmu_tsb_ctx0 + .type sun4v_mmu_tsb_ctx0,#function +sun4v_mmu_tsb_ctx0: + mov HV_FAST_MMU_TSB_CTX0, %o5 + ta HV_FAST_TRAP + retl + nop + .size sun4v_mmu_tsb_ctx0, .-sun4v_mmu_tsb_ctx0 + + /* %o0: API group number + * %o1: pointer to unsigned long major number storage + * %o2: pointer to unsigned long minor number storage + * + * returns %o0: status + */ + .globl sun4v_get_version + .type sun4v_get_version,#function +sun4v_get_version: + mov HV_CORE_GET_VER, %o5 + mov %o1, %o3 + mov %o2, %o4 + ta HV_CORE_TRAP + stx %o1, [%o3] + retl + stx %o2, [%o4] + .size sun4v_get_version, .-sun4v_get_version + + /* %o0: API group number + * %o1: desired major number + * %o2: desired minor number + * %o3: pointer to unsigned long actual minor number storage + * + * returns %o0: status + */ + .globl sun4v_set_version + .type sun4v_set_version,#function +sun4v_set_version: + mov HV_CORE_SET_VER, %o5 + mov %o3, %o4 + ta HV_CORE_TRAP + retl + stx %o1, [%o4] + .size sun4v_set_version, .-sun4v_set_version + + /* %o0: pointer to unsigned long time + * + * returns %o0: status + */ + .globl sun4v_tod_get + .type sun4v_tod_get,#function +sun4v_tod_get: + mov %o0, %o4 + mov HV_FAST_TOD_GET, %o5 + ta HV_FAST_TRAP + stx %o1, [%o4] + retl + nop + .size sun4v_tod_get, .-sun4v_tod_get + + /* %o0: time + * + * returns %o0: status + */ + .globl sun4v_tod_set + .type sun4v_tod_set,#function +sun4v_tod_set: + mov HV_FAST_TOD_SET, %o5 + ta HV_FAST_TRAP + retl + nop + .size sun4v_tod_set, .-sun4v_tod_set + + /* %o0: pointer to unsigned long status + * + * returns %o0: signed character + */ + .globl sun4v_con_getchar + .type sun4v_con_getchar,#function +sun4v_con_getchar: + mov %o0, %o4 + mov HV_FAST_CONS_GETCHAR, %o5 + clr %o0 + clr %o1 + ta HV_FAST_TRAP + stx %o0, [%o4] + retl + sra %o1, 0, %o0 + .size sun4v_con_getchar, .-sun4v_con_getchar + + /* %o0: signed long character + * + * returns %o0: status + */ + .globl sun4v_con_putchar + .type sun4v_con_putchar,#function +sun4v_con_putchar: + mov HV_FAST_CONS_PUTCHAR, %o5 + ta HV_FAST_TRAP + retl + sra %o0, 0, %o0 + .size sun4v_con_putchar, .-sun4v_con_putchar + + /* %o0: buffer real address + * %o1: buffer size + * %o2: pointer to unsigned long bytes_read + * + * returns %o0: status + */ + .globl sun4v_con_read + .type sun4v_con_read,#function +sun4v_con_read: + mov %o2, %o4 + mov HV_FAST_CONS_READ, %o5 + ta HV_FAST_TRAP + brnz %o0, 1f + cmp %o1, -1 /* break */ + be,a,pn %icc, 1f + mov %o1, %o0 + cmp %o1, -2 /* hup */ + be,a,pn %icc, 1f + mov %o1, %o0 + stx %o1, [%o4] +1: retl + nop + .size sun4v_con_read, .-sun4v_con_read + + /* %o0: buffer real address + * %o1: buffer size + * %o2: pointer to unsigned long bytes_written + * + * returns %o0: status + */ + .globl sun4v_con_write + .type sun4v_con_write,#function +sun4v_con_write: + mov %o2, %o4 + mov HV_FAST_CONS_WRITE, %o5 + ta HV_FAST_TRAP + stx %o1, [%o4] + retl + nop + .size sun4v_con_write, .-sun4v_con_write + + /* %o0: soft state + * %o1: address of description string + * + * returns %o0: status + */ + .globl sun4v_mach_set_soft_state + .type sun4v_mach_set_soft_state,#function +sun4v_mach_set_soft_state: + mov HV_FAST_MACH_SET_SOFT_STATE, %o5 + ta HV_FAST_TRAP + retl + nop + .size sun4v_mach_set_soft_state, .-sun4v_mach_set_soft_state + + /* %o0: exit code + * + * Does not return. + */ + .globl sun4v_mach_exit + .type sun4v_mach_exit,#function +sun4v_mach_exit: + mov HV_FAST_MACH_EXIT, %o5 + ta HV_FAST_TRAP + retl + nop + .size sun4v_mach_exit, .-sun4v_mach_exit + + /* %o0: buffer real address + * %o1: buffer length + * %o2: pointer to unsigned long real_buf_len + * + * returns %o0: status + */ + .globl sun4v_mach_desc + .type sun4v_mach_desc,#function +sun4v_mach_desc: + mov %o2, %o4 + mov HV_FAST_MACH_DESC, %o5 + ta HV_FAST_TRAP + stx %o1, [%o4] + retl + nop + .size sun4v_mach_desc, .-sun4v_mach_desc + + /* %o0: new timeout in milliseconds + * %o1: pointer to unsigned long orig_timeout + * + * returns %o0: status + */ + .globl sun4v_mach_set_watchdog + .type sun4v_mach_set_watchdog,#function +sun4v_mach_set_watchdog: + mov %o1, %o4 + mov HV_FAST_MACH_SET_WATCHDOG, %o5 + ta HV_FAST_TRAP + stx %o1, [%o4] + retl + nop + .size sun4v_mach_set_watchdog, .-sun4v_mach_set_watchdog + + /* No inputs and does not return. */ + .globl sun4v_mach_sir + .type sun4v_mach_sir,#function +sun4v_mach_sir: + mov %o1, %o4 + mov HV_FAST_MACH_SIR, %o5 + ta HV_FAST_TRAP + stx %o1, [%o4] + retl + nop + .size sun4v_mach_sir, .-sun4v_mach_sir + + /* %o0: channel + * %o1: ra + * %o2: num_entries + * + * returns %o0: status + */ + .globl sun4v_ldc_tx_qconf + .type sun4v_ldc_tx_qconf,#function +sun4v_ldc_tx_qconf: + mov HV_FAST_LDC_TX_QCONF, %o5 + ta HV_FAST_TRAP + retl + nop + .size sun4v_ldc_tx_qconf, .-sun4v_ldc_tx_qconf + + /* %o0: channel + * %o1: pointer to unsigned long ra + * %o2: pointer to unsigned long num_entries + * + * returns %o0: status + */ + .globl sun4v_ldc_tx_qinfo + .type sun4v_ldc_tx_qinfo,#function +sun4v_ldc_tx_qinfo: + mov %o1, %g1 + mov %o2, %g2 + mov HV_FAST_LDC_TX_QINFO, %o5 + ta HV_FAST_TRAP + stx %o1, [%g1] + stx %o2, [%g2] + retl + nop + .size sun4v_ldc_tx_qinfo, .-sun4v_ldc_tx_qinfo + + /* %o0: channel + * %o1: pointer to unsigned long head_off + * %o2: pointer to unsigned long tail_off + * %o2: pointer to unsigned long chan_state + * + * returns %o0: status + */ + .globl sun4v_ldc_tx_get_state + .type sun4v_ldc_tx_get_state,#function +sun4v_ldc_tx_get_state: + mov %o1, %g1 + mov %o2, %g2 + mov %o3, %g3 + mov HV_FAST_LDC_TX_GET_STATE, %o5 + ta HV_FAST_TRAP + stx %o1, [%g1] + stx %o2, [%g2] + stx %o3, [%g3] + retl + nop + .size sun4v_ldc_tx_get_state, .-sun4v_ldc_tx_get_state + + /* %o0: channel + * %o1: tail_off + * + * returns %o0: status + */ + .globl sun4v_ldc_tx_set_qtail + .type sun4v_ldc_tx_set_qtail,#function +sun4v_ldc_tx_set_qtail: + mov HV_FAST_LDC_TX_SET_QTAIL, %o5 + ta HV_FAST_TRAP + retl + nop + .size sun4v_ldc_tx_set_qtail, .-sun4v_ldc_tx_set_qtail + + /* %o0: channel + * %o1: ra + * %o2: num_entries + * + * returns %o0: status + */ + .globl sun4v_ldc_rx_qconf + .type sun4v_ldc_rx_qconf,#function +sun4v_ldc_rx_qconf: + mov HV_FAST_LDC_RX_QCONF, %o5 + ta HV_FAST_TRAP + retl + nop + .size sun4v_ldc_rx_qconf, .-sun4v_ldc_rx_qconf + + /* %o0: channel + * %o1: pointer to unsigned long ra + * %o2: pointer to unsigned long num_entries + * + * returns %o0: status + */ + .globl sun4v_ldc_rx_qinfo + .type sun4v_ldc_rx_qinfo,#function +sun4v_ldc_rx_qinfo: + mov %o1, %g1 + mov %o2, %g2 + mov HV_FAST_LDC_RX_QINFO, %o5 + ta HV_FAST_TRAP + stx %o1, [%g1] + stx %o2, [%g2] + retl + nop + .size sun4v_ldc_rx_qinfo, .-sun4v_ldc_rx_qinfo + + /* %o0: channel + * %o1: pointer to unsigned long head_off + * %o2: pointer to unsigned long tail_off + * %o2: pointer to unsigned long chan_state + * + * returns %o0: status + */ + .globl sun4v_ldc_rx_get_state + .type sun4v_ldc_rx_get_state,#function +sun4v_ldc_rx_get_state: + mov %o1, %g1 + mov %o2, %g2 + mov %o3, %g3 + mov HV_FAST_LDC_RX_GET_STATE, %o5 + ta HV_FAST_TRAP + stx %o1, [%g1] + stx %o2, [%g2] + stx %o3, [%g3] + retl + nop + .size sun4v_ldc_rx_get_state, .-sun4v_ldc_rx_get_state + + /* %o0: channel + * %o1: head_off + * + * returns %o0: status + */ + .globl sun4v_ldc_rx_set_qhead + .type sun4v_ldc_rx_set_qhead,#function +sun4v_ldc_rx_set_qhead: + mov HV_FAST_LDC_RX_SET_QHEAD, %o5 + ta HV_FAST_TRAP + retl + nop + .size sun4v_ldc_rx_set_qhead, .-sun4v_ldc_rx_set_qhead + + /* %o0: channel + * %o1: ra + * %o2: num_entries + * + * returns %o0: status + */ + .globl sun4v_ldc_set_map_table + .type sun4v_ldc_set_map_table,#function +sun4v_ldc_set_map_table: + mov HV_FAST_LDC_SET_MAP_TABLE, %o5 + ta HV_FAST_TRAP + retl + nop + .size sun4v_ldc_set_map_table, .-sun4v_ldc_set_map_table + + /* %o0: channel + * %o1: pointer to unsigned long ra + * %o2: pointer to unsigned long num_entries + * + * returns %o0: status + */ + .globl sun4v_ldc_get_map_table + .type sun4v_ldc_get_map_table,#function +sun4v_ldc_get_map_table: + mov %o1, %g1 + mov %o2, %g2 + mov HV_FAST_LDC_GET_MAP_TABLE, %o5 + ta HV_FAST_TRAP + stx %o1, [%g1] + stx %o2, [%g2] + retl + nop + .size sun4v_ldc_get_map_table, .-sun4v_ldc_get_map_table + + /* %o0: channel + * %o1: dir_code + * %o2: tgt_raddr + * %o3: lcl_raddr + * %o4: len + * %o5: pointer to unsigned long actual_len + * + * returns %o0: status + */ + .globl sun4v_ldc_copy + .type sun4v_ldc_copy,#function +sun4v_ldc_copy: + mov %o5, %g1 + mov HV_FAST_LDC_COPY, %o5 + ta HV_FAST_TRAP + stx %o1, [%g1] + retl + nop + .size sun4v_ldc_copy, .-sun4v_ldc_copy + + /* %o0: channel + * %o1: cookie + * %o2: pointer to unsigned long ra + * %o3: pointer to unsigned long perm + * + * returns %o0: status + */ + .globl sun4v_ldc_mapin + .type sun4v_ldc_mapin,#function +sun4v_ldc_mapin: + mov %o2, %g1 + mov %o3, %g2 + mov HV_FAST_LDC_MAPIN, %o5 + ta HV_FAST_TRAP + stx %o1, [%g1] + stx %o2, [%g2] + retl + nop + .size sun4v_ldc_mapin, .-sun4v_ldc_mapin + + /* %o0: ra + * + * returns %o0: status + */ + .globl sun4v_ldc_unmap + .type sun4v_ldc_unmap,#function +sun4v_ldc_unmap: + mov HV_FAST_LDC_UNMAP, %o5 + ta HV_FAST_TRAP + retl + nop + .size sun4v_ldc_unmap, .-sun4v_ldc_unmap + + /* %o0: channel + * %o1: cookie + * %o2: mte_cookie + * + * returns %o0: status + */ + .globl sun4v_ldc_revoke + .type sun4v_ldc_revoke,#function +sun4v_ldc_revoke: + mov HV_FAST_LDC_REVOKE, %o5 + ta HV_FAST_TRAP + retl + nop + .size sun4v_ldc_revoke, .-sun4v_ldc_revoke + + /* %o0: device handle + * %o1: device INO + * %o2: pointer to unsigned long cookie + * + * returns %o0: status + */ + .globl sun4v_vintr_get_cookie + .type sun4v_vintr_get_cookie,#function +sun4v_vintr_get_cookie: + mov %o2, %g1 + mov HV_FAST_VINTR_GET_COOKIE, %o5 + ta HV_FAST_TRAP + stx %o1, [%g1] + retl + nop + .size sun4v_vintr_get_cookie, .-sun4v_vintr_get_cookie + + /* %o0: device handle + * %o1: device INO + * %o2: cookie + * + * returns %o0: status + */ + .globl sun4v_vintr_set_cookie + .type sun4v_vintr_set_cookie,#function +sun4v_vintr_set_cookie: + mov HV_FAST_VINTR_SET_COOKIE, %o5 + ta HV_FAST_TRAP + retl + nop + .size sun4v_vintr_set_cookie, .-sun4v_vintr_set_cookie + + /* %o0: device handle + * %o1: device INO + * %o2: pointer to unsigned long valid_state + * + * returns %o0: status + */ + .globl sun4v_vintr_get_valid + .type sun4v_vintr_get_valid,#function +sun4v_vintr_get_valid: + mov %o2, %g1 + mov HV_FAST_VINTR_GET_VALID, %o5 + ta HV_FAST_TRAP + stx %o1, [%g1] + retl + nop + .size sun4v_vintr_get_valid, .-sun4v_vintr_get_valid + + /* %o0: device handle + * %o1: device INO + * %o2: valid_state + * + * returns %o0: status + */ + .globl sun4v_vintr_set_valid + .type sun4v_vintr_set_valid,#function +sun4v_vintr_set_valid: + mov HV_FAST_VINTR_SET_VALID, %o5 + ta HV_FAST_TRAP + retl + nop + .size sun4v_vintr_set_valid, .-sun4v_vintr_set_valid + + /* %o0: device handle + * %o1: device INO + * %o2: pointer to unsigned long state + * + * returns %o0: status + */ + .globl sun4v_vintr_get_state + .type sun4v_vintr_get_state,#function +sun4v_vintr_get_state: + mov %o2, %g1 + mov HV_FAST_VINTR_GET_STATE, %o5 + ta HV_FAST_TRAP + stx %o1, [%g1] + retl + nop + .size sun4v_vintr_get_state, .-sun4v_vintr_get_state + + /* %o0: device handle + * %o1: device INO + * %o2: state + * + * returns %o0: status + */ + .globl sun4v_vintr_set_state + .type sun4v_vintr_set_state,#function +sun4v_vintr_set_state: + mov HV_FAST_VINTR_SET_STATE, %o5 + ta HV_FAST_TRAP + retl + nop + .size sun4v_vintr_set_state, .-sun4v_vintr_set_state + + /* %o0: device handle + * %o1: device INO + * %o2: pointer to unsigned long cpuid + * + * returns %o0: status + */ + .globl sun4v_vintr_get_target + .type sun4v_vintr_get_target,#function +sun4v_vintr_get_target: + mov %o2, %g1 + mov HV_FAST_VINTR_GET_TARGET, %o5 + ta HV_FAST_TRAP + stx %o1, [%g1] + retl + nop + .size sun4v_vintr_get_target, .-sun4v_vintr_get_target + + /* %o0: device handle + * %o1: device INO + * %o2: cpuid + * + * returns %o0: status + */ + .globl sun4v_vintr_set_target + .type sun4v_vintr_set_target,#function +sun4v_vintr_set_target: + mov HV_FAST_VINTR_SET_TARGET, %o5 + ta HV_FAST_TRAP + retl + nop + .size sun4v_vintr_set_target, .-sun4v_vintr_set_target + + /* %o0: NCS sub-function + * %o1: sub-function arg real-address + * %o2: sub-function arg size + * + * returns %o0: status + */ + .globl sun4v_ncs_request + .type sun4v_ncs_request,#function +sun4v_ncs_request: + mov HV_FAST_NCS_REQUEST, %o5 + ta HV_FAST_TRAP + retl + nop + .size sun4v_ncs_request, .-sun4v_ncs_request + + .globl sun4v_svc_send + .type sun4v_svc_send,#function +sun4v_svc_send: + save %sp, -192, %sp + mov %i0, %o0 + mov %i1, %o1 + mov %i2, %o2 + mov HV_FAST_SVC_SEND, %o5 + ta HV_FAST_TRAP + stx %o1, [%i3] + ret + restore + .size sun4v_svc_send, .-sun4v_svc_send + + .globl sun4v_svc_recv + .type sun4v_svc_recv,#function +sun4v_svc_recv: + save %sp, -192, %sp + mov %i0, %o0 + mov %i1, %o1 + mov %i2, %o2 + mov HV_FAST_SVC_RECV, %o5 + ta HV_FAST_TRAP + stx %o1, [%i3] + ret + restore + .size sun4v_svc_recv, .-sun4v_svc_recv + + .globl sun4v_svc_getstatus + .type sun4v_svc_getstatus,#function +sun4v_svc_getstatus: + mov HV_FAST_SVC_GETSTATUS, %o5 + mov %o1, %o4 + ta HV_FAST_TRAP + stx %o1, [%o4] + retl + nop + .size sun4v_svc_getstatus, .-sun4v_svc_getstatus + + .globl sun4v_svc_setstatus + .type sun4v_svc_setstatus,#function +sun4v_svc_setstatus: + mov HV_FAST_SVC_SETSTATUS, %o5 + ta HV_FAST_TRAP + retl + nop + .size sun4v_svc_setstatus, .-sun4v_svc_setstatus + + .globl sun4v_svc_clrstatus + .type sun4v_svc_clrstatus,#function +sun4v_svc_clrstatus: + mov HV_FAST_SVC_CLRSTATUS, %o5 + ta HV_FAST_TRAP + retl + nop + .size sun4v_svc_clrstatus, .-sun4v_svc_clrstatus + + .globl sun4v_mmustat_conf + .type sun4v_mmustat_conf,#function +sun4v_mmustat_conf: + mov %o1, %o4 + mov HV_FAST_MMUSTAT_CONF, %o5 + ta HV_FAST_TRAP + stx %o1, [%o4] + retl + nop + .size sun4v_mmustat_conf, .-sun4v_mmustat_conf + + .globl sun4v_mmustat_info + .type sun4v_mmustat_info,#function +sun4v_mmustat_info: + mov %o0, %o4 + mov HV_FAST_MMUSTAT_INFO, %o5 + ta HV_FAST_TRAP + stx %o1, [%o4] + retl + nop + .size sun4v_mmustat_info, .-sun4v_mmustat_info + + .globl sun4v_mmu_demap_all + .type sun4v_mmu_demap_all,#function +sun4v_mmu_demap_all: + clr %o0 + clr %o1 + mov HV_MMU_ALL, %o2 + mov HV_FAST_MMU_DEMAP_ALL, %o5 + ta HV_FAST_TRAP + retl + nop + .size sun4v_mmu_demap_all, .-sun4v_mmu_demap_all diff --git a/arch/sparc64/kernel/iommu.c b/arch/sparc64/kernel/iommu.c index 756fa24eeef..2a37a6ca2a1 100644 --- a/arch/sparc64/kernel/iommu.c +++ b/arch/sparc64/kernel/iommu.c @@ -173,9 +173,11 @@ void iommu_range_free(struct iommu *iommu, dma_addr_t dma_addr, unsigned long np } int iommu_table_init(struct iommu *iommu, int tsbsize, - u32 dma_offset, u32 dma_addr_mask) + u32 dma_offset, u32 dma_addr_mask, + int numa_node) { - unsigned long i, tsbbase, order, sz, num_tsb_entries; + unsigned long i, order, sz, num_tsb_entries; + struct page *page; num_tsb_entries = tsbsize / sizeof(iopte_t); @@ -188,11 +190,12 @@ int iommu_table_init(struct iommu *iommu, int tsbsize, /* Allocate and initialize the free area map. */ sz = num_tsb_entries / 8; sz = (sz + 7UL) & ~7UL; - iommu->arena.map = kzalloc(sz, GFP_KERNEL); + iommu->arena.map = kmalloc_node(sz, GFP_KERNEL, numa_node); if (!iommu->arena.map) { printk(KERN_ERR "IOMMU: Error, kmalloc(arena.map) failed.\n"); return -ENOMEM; } + memset(iommu->arena.map, 0, sz); iommu->arena.limit = num_tsb_entries; if (tlb_type != hypervisor) @@ -201,21 +204,23 @@ int iommu_table_init(struct iommu *iommu, int tsbsize, /* Allocate and initialize the dummy page which we * set inactive IO PTEs to point to. */ - iommu->dummy_page = get_zeroed_page(GFP_KERNEL); - if (!iommu->dummy_page) { + page = alloc_pages_node(numa_node, GFP_KERNEL, 0); + if (!page) { printk(KERN_ERR "IOMMU: Error, gfp(dummy_page) failed.\n"); goto out_free_map; } + iommu->dummy_page = (unsigned long) page_address(page); + memset((void *)iommu->dummy_page, 0, PAGE_SIZE); iommu->dummy_page_pa = (unsigned long) __pa(iommu->dummy_page); /* Now allocate and setup the IOMMU page table itself. */ order = get_order(tsbsize); - tsbbase = __get_free_pages(GFP_KERNEL, order); - if (!tsbbase) { + page = alloc_pages_node(numa_node, GFP_KERNEL, order); + if (!page) { printk(KERN_ERR "IOMMU: Error, gfp(tsb) failed.\n"); goto out_free_dummy_page; } - iommu->page_table = (iopte_t *)tsbbase; + iommu->page_table = (iopte_t *)page_address(page); for (i = 0; i < num_tsb_entries; i++) iopte_make_dummy(iommu, &iommu->page_table[i]); @@ -276,20 +281,24 @@ static inline void iommu_free_ctx(struct iommu *iommu, int ctx) static void *dma_4u_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addrp, gfp_t gfp) { + unsigned long flags, order, first_page; struct iommu *iommu; + struct page *page; + int npages, nid; iopte_t *iopte; - unsigned long flags, order, first_page; void *ret; - int npages; size = IO_PAGE_ALIGN(size); order = get_order(size); if (order >= 10) return NULL; - first_page = __get_free_pages(gfp, order); - if (first_page == 0UL) + nid = dev->archdata.numa_node; + page = alloc_pages_node(nid, gfp, order); + if (unlikely(!page)) return NULL; + + first_page = (unsigned long) page_address(page); memset((char *)first_page, 0, PAGE_SIZE << order); iommu = dev->archdata.iommu; diff --git a/arch/sparc64/kernel/irq.c b/arch/sparc64/kernel/irq.c index eb88bd6e674..b441a26b73b 100644 --- a/arch/sparc64/kernel/irq.c +++ b/arch/sparc64/kernel/irq.c @@ -1,6 +1,6 @@ /* irq.c: UltraSparc IRQ handling/init/registry. * - * Copyright (C) 1997, 2007 David S. Miller (davem@davemloft.net) + * Copyright (C) 1997, 2007, 2008 David S. Miller (davem@davemloft.net) * Copyright (C) 1998 Eddie C. Dost (ecd@skynet.be) * Copyright (C) 1998 Jakub Jelinek (jj@ultra.linux.cz) */ @@ -308,6 +308,7 @@ static void sun4u_irq_enable(unsigned int virt_irq) IMAP_AID_SAFARI | IMAP_NID_SAFARI); val |= tid | IMAP_VALID; upa_writeq(val, imap); + upa_writeq(ICLR_IDLE, data->iclr); } } diff --git a/arch/sparc64/kernel/isa.c b/arch/sparc64/kernel/isa.c deleted file mode 100644 index b5f7b354084..00000000000 --- a/arch/sparc64/kernel/isa.c +++ /dev/null @@ -1,190 +0,0 @@ -#include <linux/kernel.h> -#include <linux/init.h> -#include <linux/pci.h> -#include <linux/slab.h> -#include <asm/oplib.h> -#include <asm/prom.h> -#include <asm/of_device.h> -#include <asm/isa.h> - -struct sparc_isa_bridge *isa_chain; - -static void __init fatal_err(const char *reason) -{ - prom_printf("ISA: fatal error, %s.\n", reason); -} - -static void __init report_dev(struct sparc_isa_device *isa_dev, int child) -{ - if (child) - printk(" (%s)", isa_dev->prom_node->name); - else - printk(" [%s", isa_dev->prom_node->name); -} - -static void __init isa_dev_get_resource(struct sparc_isa_device *isa_dev) -{ - struct of_device *op = of_find_device_by_node(isa_dev->prom_node); - - memcpy(&isa_dev->resource, &op->resource[0], sizeof(struct resource)); -} - -static void __init isa_dev_get_irq(struct sparc_isa_device *isa_dev) -{ - struct of_device *op = of_find_device_by_node(isa_dev->prom_node); - - if (!op || !op->num_irqs) { - isa_dev->irq = PCI_IRQ_NONE; - } else { - isa_dev->irq = op->irqs[0]; - } -} - -static void __init isa_fill_children(struct sparc_isa_device *parent_isa_dev) -{ - struct device_node *dp = parent_isa_dev->prom_node->child; - - if (!dp) - return; - - printk(" ->"); - while (dp) { - struct sparc_isa_device *isa_dev; - - isa_dev = kzalloc(sizeof(*isa_dev), GFP_KERNEL); - if (!isa_dev) { - fatal_err("cannot allocate child isa_dev"); - prom_halt(); - } - - /* Link it in to parent. */ - isa_dev->next = parent_isa_dev->child; - parent_isa_dev->child = isa_dev; - - isa_dev->bus = parent_isa_dev->bus; - isa_dev->prom_node = dp; - - isa_dev_get_resource(isa_dev); - isa_dev_get_irq(isa_dev); - - report_dev(isa_dev, 1); - - dp = dp->sibling; - } -} - -static void __init isa_fill_devices(struct sparc_isa_bridge *isa_br) -{ - struct device_node *dp = isa_br->prom_node->child; - - while (dp) { - struct sparc_isa_device *isa_dev; - struct dev_archdata *sd; - - isa_dev = kzalloc(sizeof(*isa_dev), GFP_KERNEL); - if (!isa_dev) { - printk(KERN_DEBUG "ISA: cannot allocate isa_dev"); - return; - } - - sd = &isa_dev->ofdev.dev.archdata; - sd->prom_node = dp; - sd->op = &isa_dev->ofdev; - sd->iommu = isa_br->ofdev.dev.parent->archdata.iommu; - sd->stc = isa_br->ofdev.dev.parent->archdata.stc; - - isa_dev->ofdev.node = dp; - isa_dev->ofdev.dev.parent = &isa_br->ofdev.dev; - isa_dev->ofdev.dev.bus = &isa_bus_type; - sprintf(isa_dev->ofdev.dev.bus_id, "isa[%08x]", dp->node); - - /* Register with core */ - if (of_device_register(&isa_dev->ofdev) != 0) { - printk(KERN_DEBUG "isa: device registration error for %s!\n", - dp->path_component_name); - kfree(isa_dev); - goto next_sibling; - } - - /* Link it in. */ - isa_dev->next = NULL; - if (isa_br->devices == NULL) { - isa_br->devices = isa_dev; - } else { - struct sparc_isa_device *tmp = isa_br->devices; - - while (tmp->next) - tmp = tmp->next; - - tmp->next = isa_dev; - } - - isa_dev->bus = isa_br; - isa_dev->prom_node = dp; - - isa_dev_get_resource(isa_dev); - isa_dev_get_irq(isa_dev); - - report_dev(isa_dev, 0); - - isa_fill_children(isa_dev); - - printk("]"); - - next_sibling: - dp = dp->sibling; - } -} - -void __init isa_init(void) -{ - struct pci_dev *pdev; - unsigned short vendor, device; - int index = 0; - - vendor = PCI_VENDOR_ID_AL; - device = PCI_DEVICE_ID_AL_M1533; - - pdev = NULL; - while ((pdev = pci_get_device(vendor, device, pdev)) != NULL) { - struct sparc_isa_bridge *isa_br; - struct device_node *dp; - - dp = pci_device_to_OF_node(pdev); - - isa_br = kzalloc(sizeof(*isa_br), GFP_KERNEL); - if (!isa_br) { - printk(KERN_DEBUG "isa: cannot allocate sparc_isa_bridge"); - pci_dev_put(pdev); - return; - } - - isa_br->ofdev.node = dp; - isa_br->ofdev.dev.parent = &pdev->dev; - isa_br->ofdev.dev.bus = &isa_bus_type; - sprintf(isa_br->ofdev.dev.bus_id, "isa%d", index); - - /* Register with core */ - if (of_device_register(&isa_br->ofdev) != 0) { - printk(KERN_DEBUG "isa: device registration error for %s!\n", - dp->path_component_name); - kfree(isa_br); - pci_dev_put(pdev); - return; - } - - /* Link it in. */ - isa_br->next = isa_chain; - isa_chain = isa_br; - - isa_br->self = pdev; - isa_br->index = index++; - isa_br->prom_node = dp; - - printk("isa%d:", isa_br->index); - - isa_fill_devices(isa_br); - - printk("\n"); - } -} diff --git a/arch/sparc64/kernel/ivec.S b/arch/sparc64/kernel/ivec.S new file mode 100644 index 00000000000..d29f92ebca5 --- /dev/null +++ b/arch/sparc64/kernel/ivec.S @@ -0,0 +1,51 @@ + /* The registers for cross calls will be: + * + * DATA 0: [low 32-bits] Address of function to call, jmp to this + * [high 32-bits] MMU Context Argument 0, place in %g5 + * DATA 1: Address Argument 1, place in %g1 + * DATA 2: Address Argument 2, place in %g7 + * + * With this method we can do most of the cross-call tlb/cache + * flushing very quickly. + */ + .align 32 + .globl do_ivec + .type do_ivec,#function +do_ivec: + mov 0x40, %g3 + ldxa [%g3 + %g0] ASI_INTR_R, %g3 + sethi %hi(KERNBASE), %g4 + cmp %g3, %g4 + bgeu,pn %xcc, do_ivec_xcall + srlx %g3, 32, %g5 + stxa %g0, [%g0] ASI_INTR_RECEIVE + membar #Sync + + sethi %hi(ivector_table_pa), %g2 + ldx [%g2 + %lo(ivector_table_pa)], %g2 + sllx %g3, 4, %g3 + add %g2, %g3, %g3 + + TRAP_LOAD_IRQ_WORK_PA(%g6, %g1) + + ldx [%g6], %g5 + stxa %g5, [%g3] ASI_PHYS_USE_EC + stx %g3, [%g6] + wr %g0, 1 << PIL_DEVICE_IRQ, %set_softint + retry +do_ivec_xcall: + mov 0x50, %g1 + ldxa [%g1 + %g0] ASI_INTR_R, %g1 + srl %g3, 0, %g3 + + mov 0x60, %g7 + ldxa [%g7 + %g0] ASI_INTR_R, %g7 + stxa %g0, [%g0] ASI_INTR_RECEIVE + membar #Sync + ba,pt %xcc, 1f + nop + + .align 32 +1: jmpl %g3, %g0 + nop + .size do_ivec,.-do_ivec diff --git a/arch/sparc64/kernel/kgdb.c b/arch/sparc64/kernel/kgdb.c new file mode 100644 index 00000000000..fefbe6dc51b --- /dev/null +++ b/arch/sparc64/kernel/kgdb.c @@ -0,0 +1,186 @@ +/* kgdb.c: KGDB support for 64-bit sparc. + * + * Copyright (C) 2008 David S. Miller <davem@davemloft.net> + */ + +#include <linux/kgdb.h> +#include <linux/kdebug.h> + +#include <asm/kdebug.h> +#include <asm/ptrace.h> +#include <asm/irq.h> + +void pt_regs_to_gdb_regs(unsigned long *gdb_regs, struct pt_regs *regs) +{ + struct reg_window *win; + int i; + + gdb_regs[GDB_G0] = 0; + for (i = 0; i < 15; i++) + gdb_regs[GDB_G1 + i] = regs->u_regs[UREG_G1 + i]; + + win = (struct reg_window *) (regs->u_regs[UREG_FP] + STACK_BIAS); + for (i = 0; i < 8; i++) + gdb_regs[GDB_L0 + i] = win->locals[i]; + for (i = 0; i < 8; i++) + gdb_regs[GDB_I0 + i] = win->ins[i]; + + for (i = GDB_F0; i <= GDB_F62; i++) + gdb_regs[i] = 0; + + gdb_regs[GDB_PC] = regs->tpc; + gdb_regs[GDB_NPC] = regs->tnpc; + gdb_regs[GDB_STATE] = regs->tstate; + gdb_regs[GDB_FSR] = 0; + gdb_regs[GDB_FPRS] = 0; + gdb_regs[GDB_Y] = regs->y; +} + +void sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *p) +{ + struct thread_info *t = task_thread_info(p); + extern unsigned int switch_to_pc; + extern unsigned int ret_from_syscall; + struct reg_window *win; + unsigned long pc, cwp; + int i; + + for (i = GDB_G0; i < GDB_G6; i++) + gdb_regs[i] = 0; + gdb_regs[GDB_G6] = (unsigned long) t; + gdb_regs[GDB_G7] = (unsigned long) p; + for (i = GDB_O0; i < GDB_SP; i++) + gdb_regs[i] = 0; + gdb_regs[GDB_SP] = t->ksp; + gdb_regs[GDB_O7] = 0; + + win = (struct reg_window *) (t->ksp + STACK_BIAS); + for (i = 0; i < 8; i++) + gdb_regs[GDB_L0 + i] = win->locals[i]; + for (i = 0; i < 8; i++) + gdb_regs[GDB_I0 + i] = win->ins[i]; + + for (i = GDB_F0; i <= GDB_F62; i++) + gdb_regs[i] = 0; + + if (t->new_child) + pc = (unsigned long) &ret_from_syscall; + else + pc = (unsigned long) &switch_to_pc; + + gdb_regs[GDB_PC] = pc; + gdb_regs[GDB_NPC] = pc + 4; + + cwp = __thread_flag_byte_ptr(t)[TI_FLAG_BYTE_CWP]; + + gdb_regs[GDB_STATE] = (TSTATE_PRIV | TSTATE_IE | cwp); + gdb_regs[GDB_FSR] = 0; + gdb_regs[GDB_FPRS] = 0; + gdb_regs[GDB_Y] = 0; +} + +void gdb_regs_to_pt_regs(unsigned long *gdb_regs, struct pt_regs *regs) +{ + struct reg_window *win; + int i; + + for (i = 0; i < 15; i++) + regs->u_regs[UREG_G1 + i] = gdb_regs[GDB_G1 + i]; + + /* If the TSTATE register is changing, we have to preserve + * the CWP field, otherwise window save/restore explodes. + */ + if (regs->tstate != gdb_regs[GDB_STATE]) { + unsigned long cwp = regs->tstate & TSTATE_CWP; + + regs->tstate = (gdb_regs[GDB_STATE] & ~TSTATE_CWP) | cwp; + } + + regs->tpc = gdb_regs[GDB_PC]; + regs->tnpc = gdb_regs[GDB_NPC]; + regs->y = gdb_regs[GDB_Y]; + + win = (struct reg_window *) (regs->u_regs[UREG_FP] + STACK_BIAS); + for (i = 0; i < 8; i++) + win->locals[i] = gdb_regs[GDB_L0 + i]; + for (i = 0; i < 8; i++) + win->ins[i] = gdb_regs[GDB_I0 + i]; +} + +#ifdef CONFIG_SMP +void smp_kgdb_capture_client(struct pt_regs *regs) +{ + unsigned long flags; + + __asm__ __volatile__("rdpr %%pstate, %0\n\t" + "wrpr %0, %1, %%pstate" + : "=r" (flags) + : "i" (PSTATE_IE)); + + flushw_all(); + + if (atomic_read(&kgdb_active) != -1) + kgdb_nmicallback(raw_smp_processor_id(), regs); + + __asm__ __volatile__("wrpr %0, 0, %%pstate" + : : "r" (flags)); +} +#endif + +int kgdb_arch_handle_exception(int e_vector, int signo, int err_code, + char *remcomInBuffer, char *remcomOutBuffer, + struct pt_regs *linux_regs) +{ + unsigned long addr; + char *ptr; + + switch (remcomInBuffer[0]) { + case 'c': + /* try to read optional parameter, pc unchanged if no parm */ + ptr = &remcomInBuffer[1]; + if (kgdb_hex2long(&ptr, &addr)) { + linux_regs->tpc = addr; + linux_regs->tnpc = addr + 4; + } + /* fallthru */ + + case 'D': + case 'k': + if (linux_regs->tpc == (unsigned long) arch_kgdb_breakpoint) { + linux_regs->tpc = linux_regs->tnpc; + linux_regs->tnpc += 4; + } + return 0; + } + return -1; +} + +asmlinkage void kgdb_trap(unsigned long trap_level, struct pt_regs *regs) +{ + unsigned long flags; + + if (user_mode(regs)) { + bad_trap(regs, trap_level); + return; + } + + flushw_all(); + + local_irq_save(flags); + kgdb_handle_exception(0x172, SIGTRAP, 0, regs); + local_irq_restore(flags); +} + +int kgdb_arch_init(void) +{ + return 0; +} + +void kgdb_arch_exit(void) +{ +} + +struct kgdb_arch arch_kgdb_ops = { + /* Breakpoint instruction: ta 0x72 */ + .gdb_bpt_instr = { 0x91, 0xd0, 0x20, 0x72 }, +}; diff --git a/arch/sparc64/kernel/mdesc.c b/arch/sparc64/kernel/mdesc.c index 91008358956..dde52bcf5c6 100644 --- a/arch/sparc64/kernel/mdesc.c +++ b/arch/sparc64/kernel/mdesc.c @@ -1,10 +1,10 @@ /* mdesc.c: Sun4V machine description handling. * - * Copyright (C) 2007 David S. Miller <davem@davemloft.net> + * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net> */ #include <linux/kernel.h> #include <linux/types.h> -#include <linux/bootmem.h> +#include <linux/lmb.h> #include <linux/log2.h> #include <linux/list.h> #include <linux/slab.h> @@ -84,24 +84,28 @@ static void mdesc_handle_init(struct mdesc_handle *hp, hp->handle_size = handle_size; } -static struct mdesc_handle * __init mdesc_bootmem_alloc(unsigned int mdesc_size) +static struct mdesc_handle * __init mdesc_lmb_alloc(unsigned int mdesc_size) { - struct mdesc_handle *hp; unsigned int handle_size, alloc_size; + struct mdesc_handle *hp; + unsigned long paddr; handle_size = (sizeof(struct mdesc_handle) - sizeof(struct mdesc_hdr) + mdesc_size); alloc_size = PAGE_ALIGN(handle_size); - hp = __alloc_bootmem(alloc_size, PAGE_SIZE, 0UL); - if (hp) - mdesc_handle_init(hp, handle_size, hp); + paddr = lmb_alloc(alloc_size, PAGE_SIZE); + hp = NULL; + if (paddr) { + hp = __va(paddr); + mdesc_handle_init(hp, handle_size, hp); + } return hp; } -static void mdesc_bootmem_free(struct mdesc_handle *hp) +static void mdesc_lmb_free(struct mdesc_handle *hp) { unsigned int alloc_size, handle_size = hp->handle_size; unsigned long start, end; @@ -124,9 +128,9 @@ static void mdesc_bootmem_free(struct mdesc_handle *hp) } } -static struct mdesc_mem_ops bootmem_mdesc_ops = { - .alloc = mdesc_bootmem_alloc, - .free = mdesc_bootmem_free, +static struct mdesc_mem_ops lmb_mdesc_ops = { + .alloc = mdesc_lmb_alloc, + .free = mdesc_lmb_free, }; static struct mdesc_handle *mdesc_kmalloc(unsigned int mdesc_size) @@ -888,7 +892,7 @@ void __init sun4v_mdesc_init(void) printk("MDESC: Size is %lu bytes.\n", len); - hp = mdesc_alloc(len, &bootmem_mdesc_ops); + hp = mdesc_alloc(len, &lmb_mdesc_ops); if (hp == NULL) { prom_printf("MDESC: alloc of %lu bytes failed.\n", len); prom_halt(); diff --git a/arch/sparc64/kernel/misctrap.S b/arch/sparc64/kernel/misctrap.S new file mode 100644 index 00000000000..753b4f031bf --- /dev/null +++ b/arch/sparc64/kernel/misctrap.S @@ -0,0 +1,97 @@ +#ifdef CONFIG_KGDB + .globl arch_kgdb_breakpoint + .type arch_kgdb_breakpoint,#function +arch_kgdb_breakpoint: + ta 0x72 + retl + nop + .size arch_kgdb_breakpoint,.-arch_kgdb_breakpoint +#endif + + .type __do_privact,#function +__do_privact: + mov TLB_SFSR, %g3 + stxa %g0, [%g3] ASI_DMMU ! Clear FaultValid bit + membar #Sync + sethi %hi(109f), %g7 + ba,pt %xcc, etrap +109: or %g7, %lo(109b), %g7 + call do_privact + add %sp, PTREGS_OFF, %o0 + ba,pt %xcc, rtrap + nop + .size __do_privact,.-__do_privact + + .type do_mna,#function +do_mna: + rdpr %tl, %g3 + cmp %g3, 1 + + /* Setup %g4/%g5 now as they are used in the + * winfixup code. + */ + mov TLB_SFSR, %g3 + mov DMMU_SFAR, %g4 + ldxa [%g4] ASI_DMMU, %g4 + ldxa [%g3] ASI_DMMU, %g5 + stxa %g0, [%g3] ASI_DMMU ! Clear FaultValid bit + membar #Sync + bgu,pn %icc, winfix_mna + rdpr %tpc, %g3 + +1: sethi %hi(109f), %g7 + ba,pt %xcc, etrap +109: or %g7, %lo(109b), %g7 + mov %l4, %o1 + mov %l5, %o2 + call mem_address_unaligned + add %sp, PTREGS_OFF, %o0 + ba,pt %xcc, rtrap + nop + .size do_mna,.-do_mna + + .type do_lddfmna,#function +do_lddfmna: + sethi %hi(109f), %g7 + mov TLB_SFSR, %g4 + ldxa [%g4] ASI_DMMU, %g5 + stxa %g0, [%g4] ASI_DMMU ! Clear FaultValid bit + membar #Sync + mov DMMU_SFAR, %g4 + ldxa [%g4] ASI_DMMU, %g4 + ba,pt %xcc, etrap +109: or %g7, %lo(109b), %g7 + mov %l4, %o1 + mov %l5, %o2 + call handle_lddfmna + add %sp, PTREGS_OFF, %o0 + ba,pt %xcc, rtrap + nop + .size do_lddfmna,.-do_lddfmna + + .type do_stdfmna,#function +do_stdfmna: + sethi %hi(109f), %g7 + mov TLB_SFSR, %g4 + ldxa [%g4] ASI_DMMU, %g5 + stxa %g0, [%g4] ASI_DMMU ! Clear FaultValid bit + membar #Sync + mov DMMU_SFAR, %g4 + ldxa [%g4] ASI_DMMU, %g4 + ba,pt %xcc, etrap +109: or %g7, %lo(109b), %g7 + mov %l4, %o1 + mov %l5, %o2 + call handle_stdfmna + add %sp, PTREGS_OFF, %o0 + ba,pt %xcc, rtrap + nop + .size do_stdfmna,.-do_stdfmna + + .type breakpoint_trap,#function +breakpoint_trap: + call sparc_breakpoint + add %sp, PTREGS_OFF, %o0 + ba,pt %xcc, rtrap + nop + .size breakpoint_trap,.-breakpoint_trap diff --git a/arch/sparc64/kernel/of_device.c b/arch/sparc64/kernel/of_device.c index 0fd9db95b89..d569f60c24b 100644 --- a/arch/sparc64/kernel/of_device.c +++ b/arch/sparc64/kernel/of_device.c @@ -6,6 +6,7 @@ #include <linux/mod_devicetable.h> #include <linux/slab.h> #include <linux/errno.h> +#include <linux/irq.h> #include <linux/of_device.h> #include <linux/of_platform.h> @@ -411,12 +412,6 @@ static int __init build_one_resource(struct device_node *parent, static int __init use_1to1_mapping(struct device_node *pp) { - /* If this is on the PMU bus, don't try to translate it even - * if a ranges property exists. - */ - if (!strcmp(pp->name, "pmu")) - return 1; - /* If we have a ranges property in the parent, use it. */ if (of_find_property(pp, "ranges", NULL) != NULL) return 0; @@ -660,6 +655,7 @@ static unsigned int __init build_one_device_irq(struct of_device *op, struct device_node *dp = op->node; struct device_node *pp, *ip; unsigned int orig_irq = irq; + int nid; if (irq == 0xffffffff) return irq; @@ -672,7 +668,7 @@ static unsigned int __init build_one_device_irq(struct of_device *op, printk("%s: direct translate %x --> %x\n", dp->full_name, orig_irq, irq); - return irq; + goto out; } /* Something more complicated. Walk up to the root, applying @@ -744,6 +740,14 @@ static unsigned int __init build_one_device_irq(struct of_device *op, printk("%s: Apply IRQ trans [%s] %x --> %x\n", op->node->full_name, ip->full_name, orig_irq, irq); +out: + nid = of_node_to_nid(dp); + if (nid != -1) { + cpumask_t numa_mask = node_to_cpumask(nid); + + irq_set_affinity(irq, numa_mask); + } + return irq; } diff --git a/arch/sparc64/kernel/pci.c b/arch/sparc64/kernel/pci.c index 545356b00e2..112b09f16f3 100644 --- a/arch/sparc64/kernel/pci.c +++ b/arch/sparc64/kernel/pci.c @@ -23,7 +23,6 @@ #include <asm/pgtable.h> #include <asm/irq.h> #include <asm/ebus.h> -#include <asm/isa.h> #include <asm/prom.h> #include <asm/apb.h> @@ -351,8 +350,7 @@ static void pci_parse_of_addrs(struct of_device *op, struct pci_dev *of_create_pci_dev(struct pci_pbm_info *pbm, struct device_node *node, - struct pci_bus *bus, int devfn, - int host_controller) + struct pci_bus *bus, int devfn) { struct dev_archdata *sd; struct pci_dev *dev; @@ -369,10 +367,12 @@ struct pci_dev *of_create_pci_dev(struct pci_pbm_info *pbm, sd->host_controller = pbm; sd->prom_node = node; sd->op = of_find_device_by_node(node); + sd->numa_node = pbm->numa_node; sd = &sd->op->dev.archdata; sd->iommu = pbm->iommu; sd->stc = &pbm->stc; + sd->numa_node = pbm->numa_node; type = of_get_property(node, "device_type", NULL); if (type == NULL) @@ -389,43 +389,28 @@ struct pci_dev *of_create_pci_dev(struct pci_pbm_info *pbm, dev->devfn = devfn; dev->multifunction = 0; /* maybe a lie? */ - if (host_controller) { - if (tlb_type != hypervisor) { - pci_read_config_word(dev, PCI_VENDOR_ID, - &dev->vendor); - pci_read_config_word(dev, PCI_DEVICE_ID, - &dev->device); - } else { - dev->vendor = PCI_VENDOR_ID_SUN; - dev->device = 0x80f0; - } - dev->cfg_size = 256; - dev->class = PCI_CLASS_BRIDGE_HOST << 8; - sprintf(pci_name(dev), "%04x:%02x:%02x.%d", pci_domain_nr(bus), - 0x00, PCI_SLOT(devfn), PCI_FUNC(devfn)); - } else { - dev->vendor = of_getintprop_default(node, "vendor-id", 0xffff); - dev->device = of_getintprop_default(node, "device-id", 0xffff); - dev->subsystem_vendor = - of_getintprop_default(node, "subsystem-vendor-id", 0); - dev->subsystem_device = - of_getintprop_default(node, "subsystem-id", 0); - - dev->cfg_size = pci_cfg_space_size(dev); - - /* We can't actually use the firmware value, we have - * to read what is in the register right now. One - * reason is that in the case of IDE interfaces the - * firmware can sample the value before the the IDE - * interface is programmed into native mode. - */ - pci_read_config_dword(dev, PCI_CLASS_REVISION, &class); - dev->class = class >> 8; - dev->revision = class & 0xff; + dev->vendor = of_getintprop_default(node, "vendor-id", 0xffff); + dev->device = of_getintprop_default(node, "device-id", 0xffff); + dev->subsystem_vendor = + of_getintprop_default(node, "subsystem-vendor-id", 0); + dev->subsystem_device = + of_getintprop_default(node, "subsystem-id", 0); + + dev->cfg_size = pci_cfg_space_size(dev); + + /* We can't actually use the firmware value, we have + * to read what is in the register right now. One + * reason is that in the case of IDE interfaces the + * firmware can sample the value before the the IDE + * interface is programmed into native mode. + */ + pci_read_config_dword(dev, PCI_CLASS_REVISION, &class); + dev->class = class >> 8; + dev->revision = class & 0xff; + + sprintf(pci_name(dev), "%04x:%02x:%02x.%d", pci_domain_nr(bus), + dev->bus->number, PCI_SLOT(devfn), PCI_FUNC(devfn)); - sprintf(pci_name(dev), "%04x:%02x:%02x.%d", pci_domain_nr(bus), - dev->bus->number, PCI_SLOT(devfn), PCI_FUNC(devfn)); - } if (ofpci_verbose) printk(" class: 0x%x device name: %s\n", dev->class, pci_name(dev)); @@ -440,26 +425,21 @@ struct pci_dev *of_create_pci_dev(struct pci_pbm_info *pbm, dev->current_state = 4; /* unknown power state */ dev->error_state = pci_channel_io_normal; - if (host_controller) { + if (!strcmp(type, "pci") || !strcmp(type, "pciex")) { + /* a PCI-PCI bridge */ dev->hdr_type = PCI_HEADER_TYPE_BRIDGE; dev->rom_base_reg = PCI_ROM_ADDRESS1; - dev->irq = PCI_IRQ_NONE; + } else if (!strcmp(type, "cardbus")) { + dev->hdr_type = PCI_HEADER_TYPE_CARDBUS; } else { - if (!strcmp(type, "pci") || !strcmp(type, "pciex")) { - /* a PCI-PCI bridge */ - dev->hdr_type = PCI_HEADER_TYPE_BRIDGE; - dev->rom_base_reg = PCI_ROM_ADDRESS1; - } else if (!strcmp(type, "cardbus")) { - dev->hdr_type = PCI_HEADER_TYPE_CARDBUS; - } else { - dev->hdr_type = PCI_HEADER_TYPE_NORMAL; - dev->rom_base_reg = PCI_ROM_ADDRESS; + dev->hdr_type = PCI_HEADER_TYPE_NORMAL; + dev->rom_base_reg = PCI_ROM_ADDRESS; - dev->irq = sd->op->irqs[0]; - if (dev->irq == 0xffffffff) - dev->irq = PCI_IRQ_NONE; - } + dev->irq = sd->op->irqs[0]; + if (dev->irq == 0xffffffff) + dev->irq = PCI_IRQ_NONE; } + pci_parse_of_addrs(sd->op, node, dev); if (ofpci_verbose) @@ -748,7 +728,7 @@ static void __devinit pci_of_scan_bus(struct pci_pbm_info *pbm, prev_devfn = devfn; /* create a new pci_dev for this device */ - dev = of_create_pci_dev(pbm, child, bus, devfn, 0); + dev = of_create_pci_dev(pbm, child, bus, devfn); if (!dev) continue; if (ofpci_verbose) @@ -795,48 +775,9 @@ static void __devinit pci_bus_register_of_sysfs(struct pci_bus *bus) pci_bus_register_of_sysfs(child_bus); } -int pci_host_bridge_read_pci_cfg(struct pci_bus *bus_dev, - unsigned int devfn, - int where, int size, - u32 *value) -{ - static u8 fake_pci_config[] = { - 0x8e, 0x10, /* Vendor: 0x108e (Sun) */ - 0xf0, 0x80, /* Device: 0x80f0 (Fire) */ - 0x46, 0x01, /* Command: 0x0146 (SERR, PARITY, MASTER, MEM) */ - 0xa0, 0x22, /* Status: 0x02a0 (DEVSEL_MED, FB2B, 66MHZ) */ - 0x00, 0x00, 0x00, 0x06, /* Class: 0x06000000 host bridge */ - 0x00, /* Cacheline: 0x00 */ - 0x40, /* Latency: 0x40 */ - 0x00, /* Header-Type: 0x00 normal */ - }; - - *value = 0; - if (where >= 0 && where < sizeof(fake_pci_config) && - (where + size) >= 0 && - (where + size) < sizeof(fake_pci_config) && - size <= sizeof(u32)) { - while (size--) { - *value <<= 8; - *value |= fake_pci_config[where + size]; - } - } - - return PCIBIOS_SUCCESSFUL; -} - -int pci_host_bridge_write_pci_cfg(struct pci_bus *bus_dev, - unsigned int devfn, - int where, int size, - u32 value) -{ - return PCIBIOS_SUCCESSFUL; -} - struct pci_bus * __devinit pci_scan_one_pbm(struct pci_pbm_info *pbm) { struct device_node *node = pbm->prom_node; - struct pci_dev *host_pdev; struct pci_bus *bus; printk("PCI: Scanning PBM %s\n", node->full_name); @@ -854,10 +795,6 @@ struct pci_bus * __devinit pci_scan_one_pbm(struct pci_pbm_info *pbm) bus->resource[0] = &pbm->io_space; bus->resource[1] = &pbm->mem_space; - /* Create the dummy host bridge and link it in. */ - host_pdev = of_create_pci_dev(pbm, node, bus, 0x00, 1); - bus->self = host_pdev; - pci_of_scan_bus(pbm, node, bus); pci_bus_add_devices(bus); pci_bus_register_of_sysfs(bus); @@ -883,7 +820,6 @@ static int __init pcibios_init(void) pci_scan_each_controller_bus(); - isa_init(); ebus_init(); power_init(); @@ -1159,6 +1095,16 @@ int pci_mmap_page_range(struct pci_dev *dev, struct vm_area_struct *vma, return 0; } +#ifdef CONFIG_NUMA +int pcibus_to_node(struct pci_bus *pbus) +{ + struct pci_pbm_info *pbm = pbus->sysdata; + + return pbm->numa_node; +} +EXPORT_SYMBOL(pcibus_to_node); +#endif + /* Return the domain nuber for this pci bus */ int pci_domain_nr(struct pci_bus *pbus) diff --git a/arch/sparc64/kernel/pci_common.c b/arch/sparc64/kernel/pci_common.c index 923e0bcc3bf..19fa621d6a6 100644 --- a/arch/sparc64/kernel/pci_common.c +++ b/arch/sparc64/kernel/pci_common.c @@ -264,9 +264,6 @@ static int sun4v_read_pci_cfg(struct pci_bus *bus_dev, unsigned int devfn, unsigned int func = PCI_FUNC(devfn); unsigned long ret; - if (!bus && devfn == 0x00) - return pci_host_bridge_read_pci_cfg(bus_dev, devfn, where, - size, value); if (config_out_of_range(pbm, bus, devfn, where)) { ret = ~0UL; } else { @@ -300,9 +297,6 @@ static int sun4v_write_pci_cfg(struct pci_bus *bus_dev, unsigned int devfn, unsigned int func = PCI_FUNC(devfn); unsigned long ret; - if (!bus && devfn == 0x00) - return pci_host_bridge_write_pci_cfg(bus_dev, devfn, where, - size, value); if (config_out_of_range(pbm, bus, devfn, where)) { /* Do nothing. */ } else { diff --git a/arch/sparc64/kernel/pci_fire.c b/arch/sparc64/kernel/pci_fire.c index 7571ed56314..d23bb6f53cd 100644 --- a/arch/sparc64/kernel/pci_fire.c +++ b/arch/sparc64/kernel/pci_fire.c @@ -71,7 +71,8 @@ static int pci_fire_pbm_iommu_init(struct pci_pbm_info *pbm) */ fire_write(iommu->iommu_flushinv, ~(u64)0); - err = iommu_table_init(iommu, tsbsize * 8 * 1024, vdma[0], dma_mask); + err = iommu_table_init(iommu, tsbsize * 8 * 1024, vdma[0], dma_mask, + pbm->numa_node); if (err) return err; @@ -449,6 +450,8 @@ static int __init pci_fire_pbm_init(struct pci_controller_info *p, pbm->next = pci_pbm_root; pci_pbm_root = pbm; + pbm->numa_node = -1; + pbm->scan_bus = pci_fire_scan_bus; pbm->pci_ops = &sun4u_pci_ops; pbm->config_space_reg_bits = 12; diff --git a/arch/sparc64/kernel/pci_impl.h b/arch/sparc64/kernel/pci_impl.h index 4a50da13ce4..c385d126be1 100644 --- a/arch/sparc64/kernel/pci_impl.h +++ b/arch/sparc64/kernel/pci_impl.h @@ -148,6 +148,8 @@ struct pci_pbm_info { struct pci_bus *pci_bus; void (*scan_bus)(struct pci_pbm_info *); struct pci_ops *pci_ops; + + int numa_node; }; struct pci_controller_info { @@ -161,21 +163,10 @@ extern struct pci_pbm_info *pci_pbm_root; extern int pci_num_pbms; /* PCI bus scanning and fixup support. */ -extern void pci_iommu_table_init(struct iommu *iommu, int tsbsize, - u32 dma_offset, u32 dma_addr_mask); extern void pci_get_pbm_props(struct pci_pbm_info *pbm); extern struct pci_bus *pci_scan_one_pbm(struct pci_pbm_info *pbm); extern void pci_determine_mem_io_space(struct pci_pbm_info *pbm); -extern int pci_host_bridge_read_pci_cfg(struct pci_bus *bus_dev, - unsigned int devfn, - int where, int size, - u32 *value); -extern int pci_host_bridge_write_pci_cfg(struct pci_bus *bus_dev, - unsigned int devfn, - int where, int size, - u32 value); - /* Error reporting support. */ extern void pci_scan_for_target_abort(struct pci_pbm_info *, struct pci_bus *); extern void pci_scan_for_master_abort(struct pci_pbm_info *, struct pci_bus *); diff --git a/arch/sparc64/kernel/pci_msi.c b/arch/sparc64/kernel/pci_msi.c index d6d64b44af6..db5e8fd8f67 100644 --- a/arch/sparc64/kernel/pci_msi.c +++ b/arch/sparc64/kernel/pci_msi.c @@ -279,11 +279,17 @@ static int bringup_one_msi_queue(struct pci_pbm_info *pbm, unsigned long devino) { int irq = ops->msiq_build_irq(pbm, msiqid, devino); - int err; + int err, nid; if (irq < 0) return irq; + nid = pbm->numa_node; + if (nid != -1) { + cpumask_t numa_mask = node_to_cpumask(nid); + + irq_set_affinity(irq, numa_mask); + } err = request_irq(irq, sparc64_msiq_interrupt, 0, "MSIQ", &pbm->msiq_irq_cookies[msiqid - pbm->msiq_first]); diff --git a/arch/sparc64/kernel/pci_psycho.c b/arch/sparc64/kernel/pci_psycho.c index 0bad96e5d18..994dbe0603d 100644 --- a/arch/sparc64/kernel/pci_psycho.c +++ b/arch/sparc64/kernel/pci_psycho.c @@ -848,7 +848,8 @@ static int psycho_iommu_init(struct pci_pbm_info *pbm) /* Leave diag mode enabled for full-flushing done * in pci_iommu.c */ - err = iommu_table_init(iommu, IO_TSB_SIZE, 0xc0000000, 0xffffffff); + err = iommu_table_init(iommu, IO_TSB_SIZE, 0xc0000000, 0xffffffff, + pbm->numa_node); if (err) return err; @@ -979,6 +980,8 @@ static void __init psycho_pbm_init(struct pci_controller_info *p, pbm->next = pci_pbm_root; pci_pbm_root = pbm; + pbm->numa_node = -1; + pbm->scan_bus = psycho_scan_bus; pbm->pci_ops = &sun4u_pci_ops; pbm->config_space_reg_bits = 8; diff --git a/arch/sparc64/kernel/pci_sabre.c b/arch/sparc64/kernel/pci_sabre.c index 1c5f5fa2339..4c34195baf3 100644 --- a/arch/sparc64/kernel/pci_sabre.c +++ b/arch/sparc64/kernel/pci_sabre.c @@ -704,7 +704,7 @@ static int sabre_iommu_init(struct pci_pbm_info *pbm, * in pci_iommu.c */ err = iommu_table_init(iommu, tsbsize * 1024 * 8, - dvma_offset, dma_mask); + dvma_offset, dma_mask, pbm->numa_node); if (err) return err; @@ -737,6 +737,8 @@ static void __init sabre_pbm_init(struct pci_controller_info *p, pbm->name = dp->full_name; printk("%s: SABRE PCI Bus Module\n", pbm->name); + pbm->numa_node = -1; + pbm->scan_bus = sabre_scan_bus; pbm->pci_ops = &sun4u_pci_ops; pbm->config_space_reg_bits = 8; diff --git a/arch/sparc64/kernel/pci_schizo.c b/arch/sparc64/kernel/pci_schizo.c index e3060936232..615edd9c8e2 100644 --- a/arch/sparc64/kernel/pci_schizo.c +++ b/arch/sparc64/kernel/pci_schizo.c @@ -1220,7 +1220,8 @@ static int schizo_pbm_iommu_init(struct pci_pbm_info *pbm) /* Leave diag mode enabled for full-flushing done * in pci_iommu.c */ - err = iommu_table_init(iommu, tsbsize * 8 * 1024, vdma[0], dma_mask); + err = iommu_table_init(iommu, tsbsize * 8 * 1024, vdma[0], dma_mask, + pbm->numa_node); if (err) return err; @@ -1379,6 +1380,8 @@ static int __init schizo_pbm_init(struct pci_controller_info *p, pbm->next = pci_pbm_root; pci_pbm_root = pbm; + pbm->numa_node = -1; + pbm->scan_bus = schizo_scan_bus; pbm->pci_ops = &sun4u_pci_ops; pbm->config_space_reg_bits = 8; diff --git a/arch/sparc64/kernel/pci_sun4v.c b/arch/sparc64/kernel/pci_sun4v.c index 01839706bd5..e2bb9790039 100644 --- a/arch/sparc64/kernel/pci_sun4v.c +++ b/arch/sparc64/kernel/pci_sun4v.c @@ -127,10 +127,12 @@ static inline long iommu_batch_end(void) static void *dma_4v_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addrp, gfp_t gfp) { - struct iommu *iommu; unsigned long flags, order, first_page, npages, n; + struct iommu *iommu; + struct page *page; void *ret; long entry; + int nid; size = IO_PAGE_ALIGN(size); order = get_order(size); @@ -139,10 +141,12 @@ static void *dma_4v_alloc_coherent(struct device *dev, size_t size, npages = size >> IO_PAGE_SHIFT; - first_page = __get_free_pages(gfp, order); - if (unlikely(first_page == 0UL)) + nid = dev->archdata.numa_node; + page = alloc_pages_node(nid, gfp, order); + if (unlikely(!page)) return NULL; + first_page = (unsigned long) page_address(page); memset((char *)first_page, 0, PAGE_SIZE << order); iommu = dev->archdata.iommu; @@ -899,6 +903,8 @@ static void __init pci_sun4v_pbm_init(struct pci_controller_info *p, pbm->next = pci_pbm_root; pci_pbm_root = pbm; + pbm->numa_node = of_node_to_nid(dp); + pbm->scan_bus = pci_sun4v_scan_bus; pbm->pci_ops = &sun4v_pci_ops; pbm->config_space_reg_bits = 12; @@ -913,6 +919,7 @@ static void __init pci_sun4v_pbm_init(struct pci_controller_info *p, pbm->name = dp->full_name; printk("%s: SUN4V PCI Bus Module\n", pbm->name); + printk("%s: On NUMA node %d\n", pbm->name, pbm->numa_node); pci_determine_mem_io_space(pbm); diff --git a/arch/sparc64/kernel/process.c b/arch/sparc64/kernel/process.c index acf8c5250aa..500ac6d483a 100644 --- a/arch/sparc64/kernel/process.c +++ b/arch/sparc64/kernel/process.c @@ -1,5 +1,4 @@ -/* $Id: process.c,v 1.131 2002/02/09 19:49:30 davem Exp $ - * arch/sparc64/kernel/process.c +/* arch/sparc64/kernel/process.c * * Copyright (C) 1995, 1996 David S. Miller (davem@caip.rutgers.edu) * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be) @@ -368,9 +367,6 @@ void flush_thread(void) if (get_thread_current_ds() != ASI_AIUS) set_fs(USER_DS); - - /* Init new signal delivery disposition. */ - clear_thread_flag(TIF_NEWSIGNALS); } /* It's a bit more tricky when 64-bit tasks are involved... */ diff --git a/arch/sparc64/kernel/prom.c b/arch/sparc64/kernel/prom.c index 68964ddcde1..ed03a18d3b3 100644 --- a/arch/sparc64/kernel/prom.c +++ b/arch/sparc64/kernel/prom.c @@ -19,8 +19,8 @@ #include <linux/types.h> #include <linux/string.h> #include <linux/mm.h> -#include <linux/bootmem.h> #include <linux/module.h> +#include <linux/lmb.h> #include <asm/prom.h> #include <asm/of_device.h> @@ -122,16 +122,20 @@ int of_find_in_proplist(const char *list, const char *match, int len) } EXPORT_SYMBOL(of_find_in_proplist); -static unsigned int prom_early_allocated; +static unsigned int prom_early_allocated __initdata; static void * __init prom_early_alloc(unsigned long size) { + unsigned long paddr = lmb_alloc(size, SMP_CACHE_BYTES); void *ret; - ret = __alloc_bootmem(size, SMP_CACHE_BYTES, 0UL); - if (ret != NULL) - memset(ret, 0, size); + if (!paddr) { + prom_printf("prom_early_alloc(%lu) failed\n"); + prom_halt(); + } + ret = __va(paddr); + memset(ret, 0, size); prom_early_allocated += size; return ret; diff --git a/arch/sparc64/kernel/rtrap.S b/arch/sparc64/kernel/rtrap.S index 079d18a11d2..ecf6753b204 100644 --- a/arch/sparc64/kernel/rtrap.S +++ b/arch/sparc64/kernel/rtrap.S @@ -18,12 +18,6 @@ #define RTRAP_PSTATE_IRQOFF (PSTATE_RMO|PSTATE_PEF|PSTATE_PRIV) #define RTRAP_PSTATE_AG_IRQOFF (PSTATE_RMO|PSTATE_PEF|PSTATE_PRIV|PSTATE_AG) - /* Register %l6 keeps track of whether we are returning - * from a system call or not. It is cleared if we call - * do_notify_resume, and it must not be otherwise modified - * until we fully commit to returning to userspace. - */ - .text .align 32 __handle_softirq: @@ -56,14 +50,12 @@ __handle_user_windows: be,pt %xcc, __handle_user_windows_continue nop mov %l5, %o1 - mov %l6, %o2 add %sp, PTREGS_OFF, %o0 - mov %l0, %o3 + mov %l0, %o2 call do_notify_resume wrpr %g0, RTRAP_PSTATE, %pstate wrpr %g0, RTRAP_PSTATE_IRQOFF, %pstate - clr %l6 /* Signal delivery can modify pt_regs tstate, so we must * reload it. */ @@ -99,14 +91,12 @@ __handle_perfctrs: be,pt %xcc, __handle_perfctrs_continue sethi %hi(TSTATE_PEF), %o0 mov %l5, %o1 - mov %l6, %o2 add %sp, PTREGS_OFF, %o0 - mov %l0, %o3 + mov %l0, %o2 call do_notify_resume wrpr %g0, RTRAP_PSTATE, %pstate wrpr %g0, RTRAP_PSTATE_IRQOFF, %pstate - clr %l6 /* Signal delivery can modify pt_regs tstate, so we must * reload it. */ @@ -127,13 +117,11 @@ __handle_userfpu: __handle_signal: mov %l5, %o1 - mov %l6, %o2 add %sp, PTREGS_OFF, %o0 - mov %l0, %o3 + mov %l0, %o2 call do_notify_resume wrpr %g0, RTRAP_PSTATE, %pstate wrpr %g0, RTRAP_PSTATE_IRQOFF, %pstate - clr %l6 /* Signal delivery can modify pt_regs tstate, so we must * reload it. @@ -145,9 +133,8 @@ __handle_signal: andn %l1, %l4, %l1 .align 64 - .globl rtrap_irq, rtrap_clr_l6, rtrap, irqsz_patchme, rtrap_xcall + .globl rtrap_irq, rtrap, irqsz_patchme, rtrap_xcall rtrap_irq: -rtrap_clr_l6: clr %l6 rtrap: #ifndef CONFIG_SMP sethi %hi(per_cpu____cpu_data), %l0 diff --git a/arch/sparc64/kernel/sbus.c b/arch/sparc64/kernel/sbus.c index d1fb13ba02b..fa2827c4a3a 100644 --- a/arch/sparc64/kernel/sbus.c +++ b/arch/sparc64/kernel/sbus.c @@ -544,6 +544,7 @@ static void __init sbus_iommu_init(int __node, struct sbus_bus *sbus) sbus->ofdev.dev.archdata.iommu = iommu; sbus->ofdev.dev.archdata.stc = strbuf; + sbus->ofdev.dev.archdata.numa_node = -1; reg_base = regs + SYSIO_IOMMUREG_BASE; iommu->iommu_control = reg_base + IOMMU_CONTROL; @@ -575,7 +576,7 @@ static void __init sbus_iommu_init(int __node, struct sbus_bus *sbus) sbus->portid, regs); /* Setup for TSB_SIZE=7, TBW_SIZE=0, MMU_DE=1, MMU_EN=1 */ - if (iommu_table_init(iommu, IO_TSB_SIZE, MAP_BASE, 0xffffffff)) + if (iommu_table_init(iommu, IO_TSB_SIZE, MAP_BASE, 0xffffffff, -1)) goto fatal_memory_error; control = upa_readq(iommu->iommu_control); diff --git a/arch/sparc64/kernel/setup.c b/arch/sparc64/kernel/setup.c index 6acb4c51cfe..da5e6ee0c66 100644 --- a/arch/sparc64/kernel/setup.c +++ b/arch/sparc64/kernel/setup.c @@ -82,7 +82,7 @@ unsigned long cmdline_memory_size = 0; static struct console prom_early_console = { .name = "earlyprom", .write = prom_console_write, - .flags = CON_PRINTBUFFER | CON_BOOT, + .flags = CON_PRINTBUFFER | CON_BOOT | CON_ANYTIME, .index = -1, }; @@ -281,6 +281,7 @@ void __init setup_arch(char **cmdline_p) /* Initialize PROM console and command line. */ *cmdline_p = prom_getbootargs(); strcpy(boot_command_line, *cmdline_p); + parse_early_param(); boot_flags_init(*cmdline_p); register_console(&prom_early_console); diff --git a/arch/sparc64/kernel/signal.c b/arch/sparc64/kernel/signal.c index 1c47009eb5e..45d6bf632da 100644 --- a/arch/sparc64/kernel/signal.c +++ b/arch/sparc64/kernel/signal.c @@ -8,7 +8,7 @@ * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) */ -#ifdef CONFIG_SPARC32_COMPAT +#ifdef CONFIG_COMPAT #include <linux/compat.h> /* for compat_old_sigset_t */ #endif #include <linux/sched.h> @@ -236,9 +236,6 @@ struct rt_signal_frame { __siginfo_fpu_t fpu_state; }; -/* Align macros */ -#define RT_ALIGNEDSZ (((sizeof(struct rt_signal_frame) + 7) & (~7))) - static long _sigpause_common(old_sigset_t set) { set &= _BLOCKABLE; @@ -335,6 +332,9 @@ void do_rt_sigreturn(struct pt_regs *regs) regs->tpc = tpc; regs->tnpc = tnpc; + /* Prevent syscall restart. */ + pt_regs_clear_trap_type(regs); + sigdelsetmask(&set, ~_BLOCKABLE); spin_lock_irq(¤t->sighand->siglock); current->blocked = set; @@ -400,7 +400,7 @@ setup_rt_frame(struct k_sigaction *ka, struct pt_regs *regs, synchronize_user_stack(); save_and_clear_fpu(); - sigframe_size = RT_ALIGNEDSZ; + sigframe_size = sizeof(struct rt_signal_frame); if (!(current_thread_info()->fpsaved[0] & FPRS_FEF)) sigframe_size -= sizeof(__siginfo_fpu_t); @@ -510,15 +510,20 @@ static inline void syscall_restart(unsigned long orig_i0, struct pt_regs *regs, * want to handle. Thus you cannot kill init even with a SIGKILL even by * mistake. */ -static void do_signal(struct pt_regs *regs, unsigned long orig_i0, int restart_syscall) +static void do_signal(struct pt_regs *regs, unsigned long orig_i0) { - siginfo_t info; struct signal_deliver_cookie cookie; struct k_sigaction ka; - int signr; sigset_t *oldset; + siginfo_t info; + int signr; - cookie.restart_syscall = restart_syscall; + if (pt_regs_is_syscall(regs) && + (regs->tstate & (TSTATE_XCARRY | TSTATE_ICARRY))) { + pt_regs_clear_trap_type(regs); + cookie.restart_syscall = 1; + } else + cookie.restart_syscall = 0; cookie.orig_i0 = orig_i0; if (test_thread_flag(TIF_RESTORE_SIGMASK)) @@ -526,12 +531,11 @@ static void do_signal(struct pt_regs *regs, unsigned long orig_i0, int restart_s else oldset = ¤t->blocked; -#ifdef CONFIG_SPARC32_COMPAT +#ifdef CONFIG_COMPAT if (test_thread_flag(TIF_32BIT)) { extern void do_signal32(sigset_t *, struct pt_regs *, - unsigned long, int); - do_signal32(oldset, regs, orig_i0, - cookie.restart_syscall); + struct signal_deliver_cookie *); + do_signal32(oldset, regs, &cookie); return; } #endif @@ -539,7 +543,7 @@ static void do_signal(struct pt_regs *regs, unsigned long orig_i0, int restart_s signr = get_signal_to_deliver(&info, &ka, regs, &cookie); if (signr > 0) { if (cookie.restart_syscall) - syscall_restart(orig_i0, regs, &ka.sa); + syscall_restart(cookie.orig_i0, regs, &ka.sa); handle_signal(signr, &ka, &info, oldset, regs); /* a signal was successfully delivered; the saved @@ -576,11 +580,10 @@ static void do_signal(struct pt_regs *regs, unsigned long orig_i0, int restart_s } } -void do_notify_resume(struct pt_regs *regs, unsigned long orig_i0, int restart_syscall, - unsigned long thread_info_flags) +void do_notify_resume(struct pt_regs *regs, unsigned long orig_i0, unsigned long thread_info_flags) { if (thread_info_flags & (_TIF_SIGPENDING | _TIF_RESTORE_SIGMASK)) - do_signal(regs, orig_i0, restart_syscall); + do_signal(regs, orig_i0); } void ptrace_signal_deliver(struct pt_regs *regs, void *cookie) diff --git a/arch/sparc64/kernel/signal32.c b/arch/sparc64/kernel/signal32.c index 74e0512f135..9415d2c918c 100644 --- a/arch/sparc64/kernel/signal32.c +++ b/arch/sparc64/kernel/signal32.c @@ -1,5 +1,4 @@ -/* $Id: signal32.c,v 1.74 2002/02/09 19:49:30 davem Exp $ - * arch/sparc64/kernel/signal32.c +/* arch/sparc64/kernel/signal32.c * * Copyright (C) 1991, 1992 Linus Torvalds * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) @@ -31,30 +30,6 @@ #define _BLOCKABLE (~(sigmask(SIGKILL) | sigmask(SIGSTOP))) -/* Signal frames: the original one (compatible with SunOS): - * - * Set up a signal frame... Make the stack look the way SunOS - * expects it to look which is basically: - * - * ---------------------------------- <-- %sp at signal time - * Struct sigcontext - * Signal address - * Ptr to sigcontext area above - * Signal code - * The signal number itself - * One register window - * ---------------------------------- <-- New %sp - */ -struct signal_sframe32 { - struct reg_window32 sig_window; - int sig_num; - int sig_code; - /* struct sigcontext32 * */ u32 sig_scptr; - int sig_address; - struct sigcontext32 sig_context; - unsigned int extramask[_COMPAT_NSIG_WORDS - 1]; -}; - /* This magic should be in g_upper[0] for all upper parts * to be valid. */ @@ -65,12 +40,7 @@ typedef struct { unsigned int asi; } siginfo_extra_v8plus_t; -/* - * And the new one, intended to be used for Linux applications only - * (we have enough in there to work with clone). - * All the interesting bits are in the info field. - */ -struct new_signal_frame32 { +struct signal_frame32 { struct sparc_stackf32 ss; __siginfo32_t info; /* __siginfo_fpu32_t * */ u32 fpu_save; @@ -149,8 +119,7 @@ struct rt_signal_frame32 { }; /* Align macros */ -#define SF_ALIGNEDSZ (((sizeof(struct signal_sframe32) + 7) & (~7))) -#define NF_ALIGNEDSZ (((sizeof(struct new_signal_frame32) + 7) & (~7))) +#define SF_ALIGNEDSZ (((sizeof(struct signal_frame32) + 7) & (~7))) #define RT_ALIGNEDSZ (((sizeof(struct rt_signal_frame32) + 7) & (~7))) int copy_siginfo_to_user32(compat_siginfo_t __user *to, siginfo_t *from) @@ -241,17 +210,22 @@ static int restore_fpu_state32(struct pt_regs *regs, __siginfo_fpu_t __user *fpu return err; } -void do_new_sigreturn32(struct pt_regs *regs) +void do_sigreturn32(struct pt_regs *regs) { - struct new_signal_frame32 __user *sf; + struct signal_frame32 __user *sf; unsigned int psr; unsigned pc, npc, fpu_save; sigset_t set; unsigned seta[_COMPAT_NSIG_WORDS]; int err, i; + /* Always make any pending restarted system calls return -EINTR */ + current_thread_info()->restart_block.fn = do_no_restart_syscall; + + synchronize_user_stack(); + regs->u_regs[UREG_FP] &= 0x00000000ffffffffUL; - sf = (struct new_signal_frame32 __user *) regs->u_regs[UREG_FP]; + sf = (struct signal_frame32 __user *) regs->u_regs[UREG_FP]; /* 1. Make sure we are not getting garbage from the user */ if (!access_ok(VERIFY_READ, sf, sizeof(*sf)) || @@ -294,6 +268,9 @@ void do_new_sigreturn32(struct pt_regs *regs) regs->tstate &= ~(TSTATE_ICC|TSTATE_XCC); regs->tstate |= psr_to_tstate_icc(psr); + /* Prevent syscall restart. */ + pt_regs_clear_trap_type(regs); + err |= __get_user(fpu_save, &sf->fpu_save); if (fpu_save) err |= restore_fpu_state32(regs, &sf->fpu_state); @@ -319,76 +296,6 @@ segv: force_sig(SIGSEGV, current); } -asmlinkage void do_sigreturn32(struct pt_regs *regs) -{ - struct sigcontext32 __user *scptr; - unsigned int pc, npc, psr; - sigset_t set; - unsigned int seta[_COMPAT_NSIG_WORDS]; - int err; - - /* Always make any pending restarted system calls return -EINTR */ - current_thread_info()->restart_block.fn = do_no_restart_syscall; - - synchronize_user_stack(); - if (test_thread_flag(TIF_NEWSIGNALS)) { - do_new_sigreturn32(regs); - return; - } - - scptr = (struct sigcontext32 __user *) - (regs->u_regs[UREG_I0] & 0x00000000ffffffffUL); - /* Check sanity of the user arg. */ - if (!access_ok(VERIFY_READ, scptr, sizeof(struct sigcontext32)) || - (((unsigned long) scptr) & 3)) - goto segv; - - err = __get_user(pc, &scptr->sigc_pc); - err |= __get_user(npc, &scptr->sigc_npc); - - if ((pc | npc) & 3) - goto segv; /* Nice try. */ - - err |= __get_user(seta[0], &scptr->sigc_mask); - /* Note that scptr + 1 points to extramask */ - err |= copy_from_user(seta+1, scptr + 1, - (_COMPAT_NSIG_WORDS - 1) * sizeof(unsigned int)); - if (err) - goto segv; - switch (_NSIG_WORDS) { - case 4: set.sig[3] = seta[6] + (((long)seta[7]) << 32); - case 3: set.sig[2] = seta[4] + (((long)seta[5]) << 32); - case 2: set.sig[1] = seta[2] + (((long)seta[3]) << 32); - case 1: set.sig[0] = seta[0] + (((long)seta[1]) << 32); - } - sigdelsetmask(&set, ~_BLOCKABLE); - spin_lock_irq(¤t->sighand->siglock); - current->blocked = set; - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); - - if (test_thread_flag(TIF_32BIT)) { - pc &= 0xffffffff; - npc &= 0xffffffff; - } - regs->tpc = pc; - regs->tnpc = npc; - err = __get_user(regs->u_regs[UREG_FP], &scptr->sigc_sp); - err |= __get_user(regs->u_regs[UREG_I0], &scptr->sigc_o0); - err |= __get_user(regs->u_regs[UREG_G1], &scptr->sigc_g1); - - /* User can only change condition codes in %tstate. */ - err |= __get_user(psr, &scptr->sigc_psr); - if (err) - goto segv; - regs->tstate &= ~(TSTATE_ICC|TSTATE_XCC); - regs->tstate |= psr_to_tstate_icc(psr); - return; - -segv: - force_sig(SIGSEGV, current); -} - asmlinkage void do_rt_sigreturn32(struct pt_regs *regs) { struct rt_signal_frame32 __user *sf; @@ -447,6 +354,9 @@ asmlinkage void do_rt_sigreturn32(struct pt_regs *regs) regs->tstate &= ~(TSTATE_ICC|TSTATE_XCC); regs->tstate |= psr_to_tstate_icc(psr); + /* Prevent syscall restart. */ + pt_regs_clear_trap_type(regs); + err |= __get_user(fpu_save, &sf->fpu_save); if (fpu_save) err |= restore_fpu_state32(regs, &sf->fpu_state); @@ -504,145 +414,6 @@ static void __user *get_sigframe(struct sigaction *sa, struct pt_regs *regs, uns return (void __user *)(sp - framesize); } -static void -setup_frame32(struct sigaction *sa, struct pt_regs *regs, int signr, sigset_t *oldset, siginfo_t *info) -{ - struct signal_sframe32 __user *sframep; - struct sigcontext32 __user *sc; - unsigned int seta[_COMPAT_NSIG_WORDS]; - int err = 0; - void __user *sig_address; - int sig_code; - unsigned long pc = regs->tpc; - unsigned long npc = regs->tnpc; - unsigned int psr; - - if (test_thread_flag(TIF_32BIT)) { - pc &= 0xffffffff; - npc &= 0xffffffff; - } - - synchronize_user_stack(); - save_and_clear_fpu(); - - sframep = (struct signal_sframe32 __user *) - get_sigframe(sa, regs, SF_ALIGNEDSZ); - if (invalid_frame_pointer(sframep, sizeof(*sframep))){ - /* Don't change signal code and address, so that - * post mortem debuggers can have a look. - */ - do_exit(SIGILL); - } - - sc = &sframep->sig_context; - - /* We've already made sure frame pointer isn't in kernel space... */ - err = __put_user((sas_ss_flags(regs->u_regs[UREG_FP]) == SS_ONSTACK), - &sc->sigc_onstack); - - switch (_NSIG_WORDS) { - case 4: seta[7] = (oldset->sig[3] >> 32); - seta[6] = oldset->sig[3]; - case 3: seta[5] = (oldset->sig[2] >> 32); - seta[4] = oldset->sig[2]; - case 2: seta[3] = (oldset->sig[1] >> 32); - seta[2] = oldset->sig[1]; - case 1: seta[1] = (oldset->sig[0] >> 32); - seta[0] = oldset->sig[0]; - } - err |= __put_user(seta[0], &sc->sigc_mask); - err |= __copy_to_user(sframep->extramask, seta + 1, - (_COMPAT_NSIG_WORDS - 1) * sizeof(unsigned int)); - err |= __put_user(regs->u_regs[UREG_FP], &sc->sigc_sp); - err |= __put_user(pc, &sc->sigc_pc); - err |= __put_user(npc, &sc->sigc_npc); - psr = tstate_to_psr(regs->tstate); - if (current_thread_info()->fpsaved[0] & FPRS_FEF) - psr |= PSR_EF; - err |= __put_user(psr, &sc->sigc_psr); - err |= __put_user(regs->u_regs[UREG_G1], &sc->sigc_g1); - err |= __put_user(regs->u_regs[UREG_I0], &sc->sigc_o0); - err |= __put_user(get_thread_wsaved(), &sc->sigc_oswins); - - err |= copy_in_user((u32 __user *)sframep, - (u32 __user *)(regs->u_regs[UREG_FP]), - sizeof(struct reg_window32)); - - set_thread_wsaved(0); /* So process is allowed to execute. */ - err |= __put_user(signr, &sframep->sig_num); - sig_address = NULL; - sig_code = 0; - if (SI_FROMKERNEL (info) && (info->si_code & __SI_MASK) == __SI_FAULT) { - sig_address = info->si_addr; - switch (signr) { - case SIGSEGV: - switch (info->si_code) { - case SEGV_MAPERR: sig_code = SUBSIG_NOMAPPING; break; - default: sig_code = SUBSIG_PROTECTION; break; - } - break; - case SIGILL: - switch (info->si_code) { - case ILL_ILLOPC: sig_code = SUBSIG_ILLINST; break; - case ILL_PRVOPC: sig_code = SUBSIG_PRIVINST; break; - case ILL_ILLTRP: sig_code = SUBSIG_BADTRAP(info->si_trapno); break; - default: sig_code = SUBSIG_STACK; break; - } - break; - case SIGFPE: - switch (info->si_code) { - case FPE_INTDIV: sig_code = SUBSIG_IDIVZERO; break; - case FPE_INTOVF: sig_code = SUBSIG_FPINTOVFL; break; - case FPE_FLTDIV: sig_code = SUBSIG_FPDIVZERO; break; - case FPE_FLTOVF: sig_code = SUBSIG_FPOVFLOW; break; - case FPE_FLTUND: sig_code = SUBSIG_FPUNFLOW; break; - case FPE_FLTRES: sig_code = SUBSIG_FPINEXACT; break; - case FPE_FLTINV: sig_code = SUBSIG_FPOPERROR; break; - default: sig_code = SUBSIG_FPERROR; break; - } - break; - case SIGBUS: - switch (info->si_code) { - case BUS_ADRALN: sig_code = SUBSIG_ALIGNMENT; break; - case BUS_ADRERR: sig_code = SUBSIG_MISCERROR; break; - default: sig_code = SUBSIG_BUSTIMEOUT; break; - } - break; - case SIGEMT: - switch (info->si_code) { - case EMT_TAGOVF: sig_code = SUBSIG_TAG; break; - } - break; - case SIGSYS: - if (info->si_code == (__SI_FAULT|0x100)) { - /* See sys_sunos32.c */ - sig_code = info->si_trapno; - break; - } - default: - sig_address = NULL; - } - } - err |= __put_user(ptr_to_compat(sig_address), &sframep->sig_address); - err |= __put_user(sig_code, &sframep->sig_code); - err |= __put_user(ptr_to_compat(sc), &sframep->sig_scptr); - if (err) - goto sigsegv; - - regs->u_regs[UREG_FP] = (unsigned long) sframep; - regs->tpc = (unsigned long) sa->sa_handler; - regs->tnpc = (regs->tpc + 4); - if (test_thread_flag(TIF_32BIT)) { - regs->tpc &= 0xffffffff; - regs->tnpc &= 0xffffffff; - } - return; - -sigsegv: - force_sigsegv(signr, current); -} - - static int save_fpu_state32(struct pt_regs *regs, __siginfo_fpu_t __user *fpu) { unsigned long *fpregs = current_thread_info()->fpregs; @@ -663,10 +434,10 @@ static int save_fpu_state32(struct pt_regs *regs, __siginfo_fpu_t __user *fpu) return err; } -static void new_setup_frame32(struct k_sigaction *ka, struct pt_regs *regs, - int signo, sigset_t *oldset) +static void setup_frame32(struct k_sigaction *ka, struct pt_regs *regs, + int signo, sigset_t *oldset) { - struct new_signal_frame32 __user *sf; + struct signal_frame32 __user *sf; int sigframe_size; u32 psr; int i, err; @@ -676,11 +447,11 @@ static void new_setup_frame32(struct k_sigaction *ka, struct pt_regs *regs, synchronize_user_stack(); save_and_clear_fpu(); - sigframe_size = NF_ALIGNEDSZ; + sigframe_size = SF_ALIGNEDSZ; if (!(current_thread_info()->fpsaved[0] & FPRS_FEF)) sigframe_size -= sizeof(__siginfo_fpu_t); - sf = (struct new_signal_frame32 __user *) + sf = (struct signal_frame32 __user *) get_sigframe(&ka->sa, regs, sigframe_size); if (invalid_frame_pointer(sf, sigframe_size)) @@ -944,10 +715,9 @@ static inline void handle_signal32(unsigned long signr, struct k_sigaction *ka, { if (ka->sa.sa_flags & SA_SIGINFO) setup_rt_frame32(ka, regs, signr, oldset, info); - else if (test_thread_flag(TIF_NEWSIGNALS)) - new_setup_frame32(ka, regs, signr, oldset); else - setup_frame32(&ka->sa, regs, signr, oldset, info); + setup_frame32(ka, regs, signr, oldset); + spin_lock_irq(¤t->sighand->siglock); sigorsets(¤t->blocked,¤t->blocked,&ka->sa.sa_mask); if (!(ka->sa.sa_flags & SA_NOMASK)) @@ -982,20 +752,16 @@ static inline void syscall_restart32(unsigned long orig_i0, struct pt_regs *regs * mistake. */ void do_signal32(sigset_t *oldset, struct pt_regs * regs, - unsigned long orig_i0, int restart_syscall) + struct signal_deliver_cookie *cookie) { - siginfo_t info; - struct signal_deliver_cookie cookie; struct k_sigaction ka; + siginfo_t info; int signr; - cookie.restart_syscall = restart_syscall; - cookie.orig_i0 = orig_i0; - - signr = get_signal_to_deliver(&info, &ka, regs, &cookie); + signr = get_signal_to_deliver(&info, &ka, regs, cookie); if (signr > 0) { - if (cookie.restart_syscall) - syscall_restart32(orig_i0, regs, &ka.sa); + if (cookie->restart_syscall) + syscall_restart32(cookie->orig_i0, regs, &ka.sa); handle_signal32(signr, &ka, &info, oldset, regs); /* a signal was successfully delivered; the saved @@ -1007,16 +773,16 @@ void do_signal32(sigset_t *oldset, struct pt_regs * regs, clear_thread_flag(TIF_RESTORE_SIGMASK); return; } - if (cookie.restart_syscall && + if (cookie->restart_syscall && (regs->u_regs[UREG_I0] == ERESTARTNOHAND || regs->u_regs[UREG_I0] == ERESTARTSYS || regs->u_regs[UREG_I0] == ERESTARTNOINTR)) { /* replay the system call when we are done */ - regs->u_regs[UREG_I0] = cookie.orig_i0; + regs->u_regs[UREG_I0] = cookie->orig_i0; regs->tpc -= 4; regs->tnpc -= 4; } - if (cookie.restart_syscall && + if (cookie->restart_syscall && regs->u_regs[UREG_I0] == ERESTART_RESTARTBLOCK) { regs->u_regs[UREG_G1] = __NR_restart_syscall; regs->tpc -= 4; diff --git a/arch/sparc64/kernel/smp.c b/arch/sparc64/kernel/smp.c index 59f020d69d4..3aba47624df 100644 --- a/arch/sparc64/kernel/smp.c +++ b/arch/sparc64/kernel/smp.c @@ -20,7 +20,7 @@ #include <linux/cache.h> #include <linux/jiffies.h> #include <linux/profile.h> -#include <linux/bootmem.h> +#include <linux/lmb.h> #include <asm/head.h> #include <asm/ptrace.h> @@ -38,7 +38,6 @@ #include <asm/pgtable.h> #include <asm/oplib.h> #include <asm/uaccess.h> -#include <asm/timer.h> #include <asm/starfire.h> #include <asm/tlb.h> #include <asm/sections.h> @@ -866,14 +865,21 @@ void smp_call_function_client(int irq, struct pt_regs *regs) void *info = call_data->info; clear_softint(1 << irq); + + irq_enter(); + + if (!call_data->wait) { + /* let initiator proceed after getting data */ + atomic_inc(&call_data->finished); + } + + func(info); + + irq_exit(); + if (call_data->wait) { /* let initiator proceed only after completion */ - func(info); - atomic_inc(&call_data->finished); - } else { - /* let initiator proceed after getting data */ atomic_inc(&call_data->finished); - func(info); } } @@ -903,6 +909,9 @@ extern unsigned long xcall_flush_tlb_kernel_range; extern unsigned long xcall_report_regs; extern unsigned long xcall_receive_signal; extern unsigned long xcall_new_mmu_context_version; +#ifdef CONFIG_KGDB +extern unsigned long xcall_kgdb_capture; +#endif #ifdef DCACHE_ALIASING_POSSIBLE extern unsigned long xcall_flush_dcache_page_cheetah; @@ -1032,7 +1041,9 @@ void smp_receive_signal(int cpu) void smp_receive_signal_client(int irq, struct pt_regs *regs) { + irq_enter(); clear_softint(1 << irq); + irq_exit(); } void smp_new_mmu_context_version_client(int irq, struct pt_regs *regs) @@ -1040,6 +1051,8 @@ void smp_new_mmu_context_version_client(int irq, struct pt_regs *regs) struct mm_struct *mm; unsigned long flags; + irq_enter(); + clear_softint(1 << irq); /* See if we need to allocate a new TLB context because @@ -1059,6 +1072,8 @@ void smp_new_mmu_context_version_client(int irq, struct pt_regs *regs) load_secondary_context(mm); __flush_tlb_mm(CTX_HWBITS(mm->context), SECONDARY_CONTEXT); + + irq_exit(); } void smp_new_mmu_context_version(void) @@ -1066,6 +1081,13 @@ void smp_new_mmu_context_version(void) smp_cross_call(&xcall_new_mmu_context_version, 0, 0, 0); } +#ifdef CONFIG_KGDB +void kgdb_roundup_cpus(unsigned long flags) +{ + smp_cross_call(&xcall_kgdb_capture, 0, 0, 0); +} +#endif + void smp_report_regs(void) { smp_cross_call(&xcall_report_regs, 0, 0, 0); @@ -1217,6 +1239,8 @@ void smp_penguin_jailcell(int irq, struct pt_regs *regs) { clear_softint(1 << irq); + irq_enter(); + preempt_disable(); __asm__ __volatile__("flushw"); @@ -1229,6 +1253,8 @@ void smp_penguin_jailcell(int irq, struct pt_regs *regs) prom_world(0); preempt_enable(); + + irq_exit(); } /* /proc/profile writes can call this, don't __init it please. */ @@ -1431,7 +1457,7 @@ EXPORT_SYMBOL(__per_cpu_shift); void __init real_setup_per_cpu_areas(void) { - unsigned long goal, size, i; + unsigned long paddr, goal, size, i; char *ptr; /* Copy section for each CPU (we discard the original) */ @@ -1441,8 +1467,13 @@ void __init real_setup_per_cpu_areas(void) for (size = PAGE_SIZE; size < goal; size <<= 1UL) __per_cpu_shift++; - ptr = alloc_bootmem_pages(size * NR_CPUS); + paddr = lmb_alloc(size * NR_CPUS, PAGE_SIZE); + if (!paddr) { + prom_printf("Cannot allocate per-cpu memory.\n"); + prom_halt(); + } + ptr = __va(paddr); __per_cpu_base = ptr - __per_cpu_start; for (i = 0; i < NR_CPUS; i++, ptr += size) diff --git a/arch/sparc64/kernel/sparc64_ksyms.c b/arch/sparc64/kernel/sparc64_ksyms.c index 38736460b8d..8ac0b99f2c5 100644 --- a/arch/sparc64/kernel/sparc64_ksyms.c +++ b/arch/sparc64/kernel/sparc64_ksyms.c @@ -49,7 +49,6 @@ #endif #ifdef CONFIG_PCI #include <asm/ebus.h> -#include <asm/isa.h> #endif #include <asm/ns87303.h> #include <asm/timer.h> @@ -68,8 +67,6 @@ extern void *__memscan_zero(void *, size_t); extern void *__memscan_generic(void *, int, size_t); extern int __memcmp(const void *, const void *, __kernel_size_t); extern __kernel_size_t strlen(const char *); -extern void linux_sparc_syscall(void); -extern void rtrap(void); extern void show_regs(struct pt_regs *); extern void syscall_trace(struct pt_regs *, int); extern void sys_sigsuspend(void); @@ -189,7 +186,6 @@ EXPORT_SYMBOL(insw); EXPORT_SYMBOL(insl); #ifdef CONFIG_PCI EXPORT_SYMBOL(ebus_chain); -EXPORT_SYMBOL(isa_chain); EXPORT_SYMBOL(pci_alloc_consistent); EXPORT_SYMBOL(pci_free_consistent); EXPORT_SYMBOL(pci_map_single); diff --git a/arch/sparc64/kernel/spiterrs.S b/arch/sparc64/kernel/spiterrs.S new file mode 100644 index 00000000000..ef902c6f8e3 --- /dev/null +++ b/arch/sparc64/kernel/spiterrs.S @@ -0,0 +1,245 @@ + /* We need to carefully read the error status, ACK the errors, + * prevent recursive traps, and pass the information on to C + * code for logging. + * + * We pass the AFAR in as-is, and we encode the status + * information as described in asm-sparc64/sfafsr.h + */ + .type __spitfire_access_error,#function +__spitfire_access_error: + /* Disable ESTATE error reporting so that we do not take + * recursive traps and RED state the processor. + */ + stxa %g0, [%g0] ASI_ESTATE_ERROR_EN + membar #Sync + + mov UDBE_UE, %g1 + ldxa [%g0] ASI_AFSR, %g4 ! Get AFSR + + /* __spitfire_cee_trap branches here with AFSR in %g4 and + * UDBE_CE in %g1. It only clears ESTATE_ERR_CE in the ESTATE + * Error Enable register. + */ +__spitfire_cee_trap_continue: + ldxa [%g0] ASI_AFAR, %g5 ! Get AFAR + + rdpr %tt, %g3 + and %g3, 0x1ff, %g3 ! Paranoia + sllx %g3, SFSTAT_TRAP_TYPE_SHIFT, %g3 + or %g4, %g3, %g4 + rdpr %tl, %g3 + cmp %g3, 1 + mov 1, %g3 + bleu %xcc, 1f + sllx %g3, SFSTAT_TL_GT_ONE_SHIFT, %g3 + + or %g4, %g3, %g4 + + /* Read in the UDB error register state, clearing the sticky + * error bits as-needed. We only clear them if the UE bit is + * set. Likewise, __spitfire_cee_trap below will only do so + * if the CE bit is set. + * + * NOTE: UltraSparc-I/II have high and low UDB error + * registers, corresponding to the two UDB units + * present on those chips. UltraSparc-IIi only + * has a single UDB, called "SDB" in the manual. + * For IIi the upper UDB register always reads + * as zero so for our purposes things will just + * work with the checks below. + */ +1: ldxa [%g0] ASI_UDBH_ERROR_R, %g3 + and %g3, 0x3ff, %g7 ! Paranoia + sllx %g7, SFSTAT_UDBH_SHIFT, %g7 + or %g4, %g7, %g4 + andcc %g3, %g1, %g3 ! UDBE_UE or UDBE_CE + be,pn %xcc, 1f + nop + stxa %g3, [%g0] ASI_UDB_ERROR_W + membar #Sync + +1: mov 0x18, %g3 + ldxa [%g3] ASI_UDBL_ERROR_R, %g3 + and %g3, 0x3ff, %g7 ! Paranoia + sllx %g7, SFSTAT_UDBL_SHIFT, %g7 + or %g4, %g7, %g4 + andcc %g3, %g1, %g3 ! UDBE_UE or UDBE_CE + be,pn %xcc, 1f + nop + mov 0x18, %g7 + stxa %g3, [%g7] ASI_UDB_ERROR_W + membar #Sync + +1: /* Ok, now that we've latched the error state, clear the + * sticky bits in the AFSR. + */ + stxa %g4, [%g0] ASI_AFSR + membar #Sync + + rdpr %tl, %g2 + cmp %g2, 1 + rdpr %pil, %g2 + bleu,pt %xcc, 1f + wrpr %g0, 15, %pil + + ba,pt %xcc, etraptl1 + rd %pc, %g7 + + ba,pt %xcc, 2f + nop + +1: ba,pt %xcc, etrap_irq + rd %pc, %g7 + +2: +#ifdef CONFIG_TRACE_IRQFLAGS + call trace_hardirqs_off + nop +#endif + mov %l4, %o1 + mov %l5, %o2 + call spitfire_access_error + add %sp, PTREGS_OFF, %o0 + ba,pt %xcc, rtrap + nop + .size __spitfire_access_error,.-__spitfire_access_error + + /* This is the trap handler entry point for ECC correctable + * errors. They are corrected, but we listen for the trap so + * that the event can be logged. + * + * Disrupting errors are either: + * 1) single-bit ECC errors during UDB reads to system + * memory + * 2) data parity errors during write-back events + * + * As far as I can make out from the manual, the CEE trap is + * only for correctable errors during memory read accesses by + * the front-end of the processor. + * + * The code below is only for trap level 1 CEE events, as it + * is the only situation where we can safely record and log. + * For trap level >1 we just clear the CE bit in the AFSR and + * return. + * + * This is just like __spiftire_access_error above, but it + * specifically handles correctable errors. If an + * uncorrectable error is indicated in the AFSR we will branch + * directly above to __spitfire_access_error to handle it + * instead. Uncorrectable therefore takes priority over + * correctable, and the error logging C code will notice this + * case by inspecting the trap type. + */ + .type __spitfire_cee_trap,#function +__spitfire_cee_trap: + ldxa [%g0] ASI_AFSR, %g4 ! Get AFSR + mov 1, %g3 + sllx %g3, SFAFSR_UE_SHIFT, %g3 + andcc %g4, %g3, %g0 ! Check for UE + bne,pn %xcc, __spitfire_access_error + nop + + /* Ok, in this case we only have a correctable error. + * Indicate we only wish to capture that state in register + * %g1, and we only disable CE error reporting unlike UE + * handling which disables all errors. + */ + ldxa [%g0] ASI_ESTATE_ERROR_EN, %g3 + andn %g3, ESTATE_ERR_CE, %g3 + stxa %g3, [%g0] ASI_ESTATE_ERROR_EN + membar #Sync + + /* Preserve AFSR in %g4, indicate UDB state to capture in %g1 */ + ba,pt %xcc, __spitfire_cee_trap_continue + mov UDBE_CE, %g1 + .size __spitfire_cee_trap,.-__spitfire_cee_trap + + .type __spitfire_data_access_exception_tl1,#function +__spitfire_data_access_exception_tl1: + rdpr %pstate, %g4 + wrpr %g4, PSTATE_MG|PSTATE_AG, %pstate + mov TLB_SFSR, %g3 + mov DMMU_SFAR, %g5 + ldxa [%g3] ASI_DMMU, %g4 ! Get SFSR + ldxa [%g5] ASI_DMMU, %g5 ! Get SFAR + stxa %g0, [%g3] ASI_DMMU ! Clear SFSR.FaultValid bit + membar #Sync + rdpr %tt, %g3 + cmp %g3, 0x80 ! first win spill/fill trap + blu,pn %xcc, 1f + cmp %g3, 0xff ! last win spill/fill trap + bgu,pn %xcc, 1f + nop + ba,pt %xcc, winfix_dax + rdpr %tpc, %g3 +1: sethi %hi(109f), %g7 + ba,pt %xcc, etraptl1 +109: or %g7, %lo(109b), %g7 + mov %l4, %o1 + mov %l5, %o2 + call spitfire_data_access_exception_tl1 + add %sp, PTREGS_OFF, %o0 + ba,pt %xcc, rtrap + nop + .size __spitfire_data_access_exception_tl1,.-__spitfire_data_access_exception_tl1 + + .type __spitfire_data_access_exception,#function +__spitfire_data_access_exception: + rdpr %pstate, %g4 + wrpr %g4, PSTATE_MG|PSTATE_AG, %pstate + mov TLB_SFSR, %g3 + mov DMMU_SFAR, %g5 + ldxa [%g3] ASI_DMMU, %g4 ! Get SFSR + ldxa [%g5] ASI_DMMU, %g5 ! Get SFAR + stxa %g0, [%g3] ASI_DMMU ! Clear SFSR.FaultValid bit + membar #Sync + sethi %hi(109f), %g7 + ba,pt %xcc, etrap +109: or %g7, %lo(109b), %g7 + mov %l4, %o1 + mov %l5, %o2 + call spitfire_data_access_exception + add %sp, PTREGS_OFF, %o0 + ba,pt %xcc, rtrap + nop + .size __spitfire_data_access_exception,.-__spitfire_data_access_exception + + .type __spitfire_insn_access_exception_tl1,#function +__spitfire_insn_access_exception_tl1: + rdpr %pstate, %g4 + wrpr %g4, PSTATE_MG|PSTATE_AG, %pstate + mov TLB_SFSR, %g3 + ldxa [%g3] ASI_IMMU, %g4 ! Get SFSR + rdpr %tpc, %g5 ! IMMU has no SFAR, use TPC + stxa %g0, [%g3] ASI_IMMU ! Clear FaultValid bit + membar #Sync + sethi %hi(109f), %g7 + ba,pt %xcc, etraptl1 +109: or %g7, %lo(109b), %g7 + mov %l4, %o1 + mov %l5, %o2 + call spitfire_insn_access_exception_tl1 + add %sp, PTREGS_OFF, %o0 + ba,pt %xcc, rtrap + nop + .size __spitfire_insn_access_exception_tl1,.-__spitfire_insn_access_exception_tl1 + + .type __spitfire_insn_access_exception,#function +__spitfire_insn_access_exception: + rdpr %pstate, %g4 + wrpr %g4, PSTATE_MG|PSTATE_AG, %pstate + mov TLB_SFSR, %g3 + ldxa [%g3] ASI_IMMU, %g4 ! Get SFSR + rdpr %tpc, %g5 ! IMMU has no SFAR, use TPC + stxa %g0, [%g3] ASI_IMMU ! Clear FaultValid bit + membar #Sync + sethi %hi(109f), %g7 + ba,pt %xcc, etrap +109: or %g7, %lo(109b), %g7 + mov %l4, %o1 + mov %l5, %o2 + call spitfire_insn_access_exception + add %sp, PTREGS_OFF, %o0 + ba,pt %xcc, rtrap + nop + .size __spitfire_insn_access_exception,.-__spitfire_insn_access_exception diff --git a/arch/sparc64/kernel/stacktrace.c b/arch/sparc64/kernel/stacktrace.c index 84d39e873e8..01b52f561af 100644 --- a/arch/sparc64/kernel/stacktrace.c +++ b/arch/sparc64/kernel/stacktrace.c @@ -20,6 +20,8 @@ void save_stack_trace(struct stack_trace *trace) thread_base = (unsigned long) tp; do { struct reg_window *rw; + struct pt_regs *regs; + unsigned long pc; /* Bogus frame pointer? */ if (fp < (thread_base + sizeof(struct thread_info)) || @@ -27,11 +29,19 @@ void save_stack_trace(struct stack_trace *trace) break; rw = (struct reg_window *) fp; + regs = (struct pt_regs *) (rw + 1); + + if ((regs->magic & ~0x1ff) == PT_REGS_MAGIC) { + pc = regs->tpc; + fp = regs->u_regs[UREG_I6] + STACK_BIAS; + } else { + pc = rw->ins[7]; + fp = rw->ins[6] + STACK_BIAS; + } + if (trace->skip > 0) trace->skip--; else - trace->entries[trace->nr_entries++] = rw->ins[7]; - - fp = rw->ins[6] + STACK_BIAS; + trace->entries[trace->nr_entries++] = pc; } while (trace->nr_entries < trace->max_entries); } diff --git a/arch/sparc64/kernel/sun4v_tlb_miss.S b/arch/sparc64/kernel/sun4v_tlb_miss.S index fd9430562e0..e1fbf8c7578 100644 --- a/arch/sparc64/kernel/sun4v_tlb_miss.S +++ b/arch/sparc64/kernel/sun4v_tlb_miss.S @@ -262,7 +262,7 @@ sun4v_iacc: mov %l5, %o2 call sun4v_insn_access_exception add %sp, PTREGS_OFF, %o0 - ba,a,pt %xcc, rtrap_clr_l6 + ba,a,pt %xcc, rtrap /* Instruction Access Exception, tl1. */ sun4v_iacc_tl1: @@ -278,7 +278,7 @@ sun4v_iacc_tl1: mov %l5, %o2 call sun4v_insn_access_exception_tl1 add %sp, PTREGS_OFF, %o0 - ba,a,pt %xcc, rtrap_clr_l6 + ba,a,pt %xcc, rtrap /* Data Access Exception, tl0. */ sun4v_dacc: @@ -294,7 +294,7 @@ sun4v_dacc: mov %l5, %o2 call sun4v_data_access_exception add %sp, PTREGS_OFF, %o0 - ba,a,pt %xcc, rtrap_clr_l6 + ba,a,pt %xcc, rtrap /* Data Access Exception, tl1. */ sun4v_dacc_tl1: @@ -310,7 +310,7 @@ sun4v_dacc_tl1: mov %l5, %o2 call sun4v_data_access_exception_tl1 add %sp, PTREGS_OFF, %o0 - ba,a,pt %xcc, rtrap_clr_l6 + ba,a,pt %xcc, rtrap /* Memory Address Unaligned. */ sun4v_mna: @@ -344,7 +344,7 @@ sun4v_mna: mov %l5, %o2 call sun4v_do_mna add %sp, PTREGS_OFF, %o0 - ba,a,pt %xcc, rtrap_clr_l6 + ba,a,pt %xcc, rtrap /* Privileged Action. */ sun4v_privact: @@ -352,7 +352,7 @@ sun4v_privact: rd %pc, %g7 call do_privact add %sp, PTREGS_OFF, %o0 - ba,a,pt %xcc, rtrap_clr_l6 + ba,a,pt %xcc, rtrap /* Unaligned ldd float, tl0. */ sun4v_lddfmna: @@ -368,7 +368,7 @@ sun4v_lddfmna: mov %l5, %o2 call handle_lddfmna add %sp, PTREGS_OFF, %o0 - ba,a,pt %xcc, rtrap_clr_l6 + ba,a,pt %xcc, rtrap /* Unaligned std float, tl0. */ sun4v_stdfmna: @@ -384,7 +384,7 @@ sun4v_stdfmna: mov %l5, %o2 call handle_stdfmna add %sp, PTREGS_OFF, %o0 - ba,a,pt %xcc, rtrap_clr_l6 + ba,a,pt %xcc, rtrap #define BRANCH_ALWAYS 0x10680000 #define NOP 0x01000000 diff --git a/arch/sparc64/kernel/sys_sparc.c b/arch/sparc64/kernel/sys_sparc.c index 73ed01ba40d..8d4761f15fa 100644 --- a/arch/sparc64/kernel/sys_sparc.c +++ b/arch/sparc64/kernel/sys_sparc.c @@ -454,8 +454,8 @@ asmlinkage long sys_ipc(unsigned int call, int first, unsigned long second, err = sys_semget(first, (int)second, (int)third); goto out; case SEMCTL: { - err = sys_semctl(first, third, - (int)second | IPC_64, + err = sys_semctl(first, second, + (int)third | IPC_64, (union semun) ptr); goto out; } diff --git a/arch/sparc64/kernel/sys_sparc32.c b/arch/sparc64/kernel/sys_sparc32.c index c1a61e98899..161ce4710fe 100644 --- a/arch/sparc64/kernel/sys_sparc32.c +++ b/arch/sparc64/kernel/sys_sparc32.c @@ -554,10 +554,8 @@ asmlinkage long compat_sys_sigaction(int sig, struct old_sigaction32 __user *act struct k_sigaction new_ka, old_ka; int ret; - if (sig < 0) { - set_thread_flag(TIF_NEWSIGNALS); - sig = -sig; - } + WARN_ON_ONCE(sig >= 0); + sig = -sig; if (act) { compat_old_sigset_t mask; @@ -601,11 +599,6 @@ asmlinkage long compat_sys_rt_sigaction(int sig, if (sigsetsize != sizeof(compat_sigset_t)) return -EINVAL; - /* All tasks which use RT signals (effectively) use - * new style signals. - */ - set_thread_flag(TIF_NEWSIGNALS); - if (act) { u32 u_handler, u_restorer; diff --git a/arch/sparc64/kernel/syscalls.S b/arch/sparc64/kernel/syscalls.S new file mode 100644 index 00000000000..db19ed67acf --- /dev/null +++ b/arch/sparc64/kernel/syscalls.S @@ -0,0 +1,279 @@ + /* SunOS's execv() call only specifies the argv argument, the + * environment settings are the same as the calling processes. + */ +sys_execve: + sethi %hi(sparc_execve), %g1 + ba,pt %xcc, execve_merge + or %g1, %lo(sparc_execve), %g1 + +#ifdef CONFIG_COMPAT +sunos_execv: + stx %g0, [%sp + PTREGS_OFF + PT_V9_I2] +sys32_execve: + sethi %hi(sparc32_execve), %g1 + or %g1, %lo(sparc32_execve), %g1 +#endif + +execve_merge: + flushw + jmpl %g1, %g0 + add %sp, PTREGS_OFF, %o0 + + .align 32 +sys_pipe: + ba,pt %xcc, sparc_pipe + add %sp, PTREGS_OFF, %o0 +sys_nis_syscall: + ba,pt %xcc, c_sys_nis_syscall + add %sp, PTREGS_OFF, %o0 +sys_memory_ordering: + ba,pt %xcc, sparc_memory_ordering + add %sp, PTREGS_OFF, %o1 +sys_sigaltstack: + ba,pt %xcc, do_sigaltstack + add %i6, STACK_BIAS, %o2 +#ifdef CONFIG_COMPAT +sys32_sigstack: + ba,pt %xcc, do_sys32_sigstack + mov %i6, %o2 +sys32_sigaltstack: + ba,pt %xcc, do_sys32_sigaltstack + mov %i6, %o2 +#endif + .align 32 +#ifdef CONFIG_COMPAT +sys32_sigreturn: + add %sp, PTREGS_OFF, %o0 + call do_sigreturn32 + add %o7, 1f-.-4, %o7 + nop +#endif +sys_rt_sigreturn: + add %sp, PTREGS_OFF, %o0 + call do_rt_sigreturn + add %o7, 1f-.-4, %o7 + nop +#ifdef CONFIG_COMPAT +sys32_rt_sigreturn: + add %sp, PTREGS_OFF, %o0 + call do_rt_sigreturn32 + add %o7, 1f-.-4, %o7 + nop +#endif + .align 32 +1: ldx [%g6 + TI_FLAGS], %l5 + andcc %l5, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT), %g0 + be,pt %icc, rtrap + nop + add %sp, PTREGS_OFF, %o0 + call syscall_trace + mov 1, %o1 + ba,pt %xcc, rtrap + nop + + /* This is how fork() was meant to be done, 8 instruction entry. + * + * I questioned the following code briefly, let me clear things + * up so you must not reason on it like I did. + * + * Know the fork_kpsr etc. we use in the sparc32 port? We don't + * need it here because the only piece of window state we copy to + * the child is the CWP register. Even if the parent sleeps, + * we are safe because we stuck it into pt_regs of the parent + * so it will not change. + * + * XXX This raises the question, whether we can do the same on + * XXX sparc32 to get rid of fork_kpsr _and_ fork_kwim. The + * XXX answer is yes. We stick fork_kpsr in UREG_G0 and + * XXX fork_kwim in UREG_G1 (global registers are considered + * XXX volatile across a system call in the sparc ABI I think + * XXX if it isn't we can use regs->y instead, anyone who depends + * XXX upon the Y register being preserved across a fork deserves + * XXX to lose). + * + * In fact we should take advantage of that fact for other things + * during system calls... + */ + .align 32 +sys_vfork: /* Under Linux, vfork and fork are just special cases of clone. */ + sethi %hi(0x4000 | 0x0100 | SIGCHLD), %o0 + or %o0, %lo(0x4000 | 0x0100 | SIGCHLD), %o0 + ba,pt %xcc, sys_clone +sys_fork: + clr %o1 + mov SIGCHLD, %o0 +sys_clone: + flushw + movrz %o1, %fp, %o1 + mov 0, %o3 + ba,pt %xcc, sparc_do_fork + add %sp, PTREGS_OFF, %o2 + + .globl ret_from_syscall +ret_from_syscall: + /* Clear current_thread_info()->new_child, and + * check performance counter stuff too. + */ + stb %g0, [%g6 + TI_NEW_CHILD] + ldx [%g6 + TI_FLAGS], %l0 + call schedule_tail + mov %g7, %o0 + andcc %l0, _TIF_PERFCTR, %g0 + be,pt %icc, 1f + nop + ldx [%g6 + TI_PCR], %o7 + wr %g0, %o7, %pcr + + /* Blackbird errata workaround. See commentary in + * smp.c:smp_percpu_timer_interrupt() for more + * information. + */ + ba,pt %xcc, 99f + nop + + .align 64 +99: wr %g0, %g0, %pic + rd %pic, %g0 + +1: ba,pt %xcc, ret_sys_call + ldx [%sp + PTREGS_OFF + PT_V9_I0], %o0 + + .globl sparc_exit + .type sparc_exit,#function +sparc_exit: + rdpr %pstate, %g2 + wrpr %g2, PSTATE_IE, %pstate + rdpr %otherwin, %g1 + rdpr %cansave, %g3 + add %g3, %g1, %g3 + wrpr %g3, 0x0, %cansave + wrpr %g0, 0x0, %otherwin + wrpr %g2, 0x0, %pstate + ba,pt %xcc, sys_exit + stb %g0, [%g6 + TI_WSAVED] + .size sparc_exit,.-sparc_exit + +linux_sparc_ni_syscall: + sethi %hi(sys_ni_syscall), %l7 + ba,pt %xcc, 4f + or %l7, %lo(sys_ni_syscall), %l7 + +linux_syscall_trace32: + add %sp, PTREGS_OFF, %o0 + call syscall_trace + clr %o1 + srl %i0, 0, %o0 + srl %i4, 0, %o4 + srl %i1, 0, %o1 + srl %i2, 0, %o2 + ba,pt %xcc, 2f + srl %i3, 0, %o3 + +linux_syscall_trace: + add %sp, PTREGS_OFF, %o0 + call syscall_trace + clr %o1 + mov %i0, %o0 + mov %i1, %o1 + mov %i2, %o2 + mov %i3, %o3 + b,pt %xcc, 2f + mov %i4, %o4 + + + /* Linux 32-bit system calls enter here... */ + .align 32 + .globl linux_sparc_syscall32 +linux_sparc_syscall32: + /* Direct access to user regs, much faster. */ + cmp %g1, NR_SYSCALLS ! IEU1 Group + bgeu,pn %xcc, linux_sparc_ni_syscall ! CTI + srl %i0, 0, %o0 ! IEU0 + sll %g1, 2, %l4 ! IEU0 Group + srl %i4, 0, %o4 ! IEU1 + lduw [%l7 + %l4], %l7 ! Load + srl %i1, 0, %o1 ! IEU0 Group + ldx [%g6 + TI_FLAGS], %l0 ! Load + + srl %i5, 0, %o5 ! IEU1 + srl %i2, 0, %o2 ! IEU0 Group + andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT), %g0 + bne,pn %icc, linux_syscall_trace32 ! CTI + mov %i0, %l5 ! IEU1 + call %l7 ! CTI Group brk forced + srl %i3, 0, %o3 ! IEU0 + ba,a,pt %xcc, 3f + + /* Linux native system calls enter here... */ + .align 32 + .globl linux_sparc_syscall +linux_sparc_syscall: + /* Direct access to user regs, much faster. */ + cmp %g1, NR_SYSCALLS ! IEU1 Group + bgeu,pn %xcc, linux_sparc_ni_syscall ! CTI + mov %i0, %o0 ! IEU0 + sll %g1, 2, %l4 ! IEU0 Group + mov %i1, %o1 ! IEU1 + lduw [%l7 + %l4], %l7 ! Load +4: mov %i2, %o2 ! IEU0 Group + ldx [%g6 + TI_FLAGS], %l0 ! Load + + mov %i3, %o3 ! IEU1 + mov %i4, %o4 ! IEU0 Group + andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT), %g0 + bne,pn %icc, linux_syscall_trace ! CTI Group + mov %i0, %l5 ! IEU0 +2: call %l7 ! CTI Group brk forced + mov %i5, %o5 ! IEU0 + nop + +3: stx %o0, [%sp + PTREGS_OFF + PT_V9_I0] +ret_sys_call: + ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %g3 + ldx [%sp + PTREGS_OFF + PT_V9_TNPC], %l1 ! pc = npc + sra %o0, 0, %o0 + mov %ulo(TSTATE_XCARRY | TSTATE_ICARRY), %g2 + sllx %g2, 32, %g2 + + /* Check if force_successful_syscall_return() + * was invoked. + */ + ldub [%g6 + TI_SYS_NOERROR], %l2 + brnz,a,pn %l2, 80f + stb %g0, [%g6 + TI_SYS_NOERROR] + + cmp %o0, -ERESTART_RESTARTBLOCK + bgeu,pn %xcc, 1f + andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT), %l6 +80: + /* System call success, clear Carry condition code. */ + andn %g3, %g2, %g3 + stx %g3, [%sp + PTREGS_OFF + PT_V9_TSTATE] + bne,pn %icc, linux_syscall_trace2 + add %l1, 0x4, %l2 ! npc = npc+4 + stx %l1, [%sp + PTREGS_OFF + PT_V9_TPC] + ba,pt %xcc, rtrap + stx %l2, [%sp + PTREGS_OFF + PT_V9_TNPC] + +1: + /* System call failure, set Carry condition code. + * Also, get abs(errno) to return to the process. + */ + andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT), %l6 + sub %g0, %o0, %o0 + or %g3, %g2, %g3 + stx %o0, [%sp + PTREGS_OFF + PT_V9_I0] + stx %g3, [%sp + PTREGS_OFF + PT_V9_TSTATE] + bne,pn %icc, linux_syscall_trace2 + add %l1, 0x4, %l2 ! npc = npc+4 + stx %l1, [%sp + PTREGS_OFF + PT_V9_TPC] + + b,pt %xcc, rtrap + stx %l2, [%sp + PTREGS_OFF + PT_V9_TNPC] +linux_syscall_trace2: + add %sp, PTREGS_OFF, %o0 + call syscall_trace + mov 1, %o1 + stx %l1, [%sp + PTREGS_OFF + PT_V9_TPC] + ba,pt %xcc, rtrap + stx %l2, [%sp + PTREGS_OFF + PT_V9_TNPC] diff --git a/arch/sparc64/kernel/sysfs.c b/arch/sparc64/kernel/sysfs.c index 52816c7be0b..e885034a6b7 100644 --- a/arch/sparc64/kernel/sysfs.c +++ b/arch/sparc64/kernel/sysfs.c @@ -273,10 +273,22 @@ static void __init check_mmu_stats(void) mmu_stats_supported = 1; } +static void register_nodes(void) +{ +#ifdef CONFIG_NUMA + int i; + + for (i = 0; i < MAX_NUMNODES; i++) + register_one_node(i); +#endif +} + static int __init topology_init(void) { int cpu; + register_nodes(); + check_mmu_stats(); register_cpu_notifier(&sysfs_cpu_nb); diff --git a/arch/sparc64/kernel/traps.c b/arch/sparc64/kernel/traps.c index 96da847023f..d9b8d46707d 100644 --- a/arch/sparc64/kernel/traps.c +++ b/arch/sparc64/kernel/traps.c @@ -2091,9 +2091,8 @@ static void user_instruction_dump(unsigned int __user *pc) void show_stack(struct task_struct *tsk, unsigned long *_ksp) { - unsigned long pc, fp, thread_base, ksp; + unsigned long fp, thread_base, ksp; struct thread_info *tp; - struct reg_window *rw; int count = 0; ksp = (unsigned long) _ksp; @@ -2117,15 +2116,27 @@ void show_stack(struct task_struct *tsk, unsigned long *_ksp) printk("\n"); #endif do { + struct reg_window *rw; + struct pt_regs *regs; + unsigned long pc; + /* Bogus frame pointer? */ if (fp < (thread_base + sizeof(struct thread_info)) || fp >= (thread_base + THREAD_SIZE)) break; rw = (struct reg_window *)fp; - pc = rw->ins[7]; + regs = (struct pt_regs *) (rw + 1); + + if ((regs->magic & ~0x1ff) == PT_REGS_MAGIC) { + pc = regs->tpc; + fp = regs->u_regs[UREG_I6] + STACK_BIAS; + } else { + pc = rw->ins[7]; + fp = rw->ins[6] + STACK_BIAS; + } + printk(" [%016lx] ", pc); print_symbol("%s\n", pc); - fp = rw->ins[6] + STACK_BIAS; } while (++count < 16); #ifndef CONFIG_KALLSYMS printk("\n"); diff --git a/arch/sparc64/kernel/tsb.S b/arch/sparc64/kernel/tsb.S index 10adb2fb8ff..c499214b501 100644 --- a/arch/sparc64/kernel/tsb.S +++ b/arch/sparc64/kernel/tsb.S @@ -275,7 +275,7 @@ sparc64_realfault_common: stx %l5, [%g6 + TI_FAULT_ADDR] ! Save fault address call do_sparc64_fault ! Call fault handler add %sp, PTREGS_OFF, %o0 ! Compute pt_regs arg - ba,pt %xcc, rtrap_clr_l6 ! Restore cpu state + ba,pt %xcc, rtrap ! Restore cpu state nop ! Delay slot (fill me) winfix_trampoline: diff --git a/arch/sparc64/kernel/ttable.S b/arch/sparc64/kernel/ttable.S index b0de4c00b11..450053af039 100644 --- a/arch/sparc64/kernel/ttable.S +++ b/arch/sparc64/kernel/ttable.S @@ -153,7 +153,7 @@ tl0_resv164: BTRAP(0x164) BTRAP(0x165) BTRAP(0x166) BTRAP(0x167) BTRAP(0x168) tl0_resv169: BTRAP(0x169) BTRAP(0x16a) BTRAP(0x16b) BTRAP(0x16c) tl0_linux64: LINUX_64BIT_SYSCALL_TRAP tl0_gsctx: TRAP(sparc64_get_context) TRAP(sparc64_set_context) -tl0_resv170: KPROBES_TRAP(0x170) KPROBES_TRAP(0x171) BTRAP(0x172) +tl0_resv170: KPROBES_TRAP(0x170) KPROBES_TRAP(0x171) KGDB_TRAP(0x172) tl0_resv173: BTRAP(0x173) BTRAP(0x174) BTRAP(0x175) BTRAP(0x176) BTRAP(0x177) tl0_resv178: BTRAP(0x178) BTRAP(0x179) BTRAP(0x17a) BTRAP(0x17b) BTRAP(0x17c) tl0_resv17d: BTRAP(0x17d) BTRAP(0x17e) BTRAP(0x17f) diff --git a/arch/sparc64/kernel/utrap.S b/arch/sparc64/kernel/utrap.S new file mode 100644 index 00000000000..b7f0f3f3a90 --- /dev/null +++ b/arch/sparc64/kernel/utrap.S @@ -0,0 +1,29 @@ + .globl utrap_trap + .type utrap_trap,#function +utrap_trap: /* %g3=handler,%g4=level */ + TRAP_LOAD_THREAD_REG(%g6, %g1) + ldx [%g6 + TI_UTRAPS], %g1 + brnz,pt %g1, invoke_utrap + nop + + ba,pt %xcc, etrap + rd %pc, %g7 + mov %l4, %o1 + call bad_trap + add %sp, PTREGS_OFF, %o0 + ba,pt %xcc, rtrap + nop + +invoke_utrap: + sllx %g3, 3, %g3 + ldx [%g1 + %g3], %g1 + save %sp, -128, %sp + rdpr %tstate, %l6 + rdpr %cwp, %l7 + andn %l6, TSTATE_CWP, %l6 + wrpr %l6, %l7, %tstate + rdpr %tpc, %l6 + rdpr %tnpc, %l7 + wrpr %g1, 0, %tnpc + done + .size utrap_trap,.-utrap_trap diff --git a/arch/sparc64/kernel/winfixup.S b/arch/sparc64/kernel/winfixup.S index c4aa110a10e..a6b0863c27d 100644 --- a/arch/sparc64/kernel/winfixup.S +++ b/arch/sparc64/kernel/winfixup.S @@ -32,7 +32,7 @@ fill_fixup: rd %pc, %g7 call do_sparc64_fault add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap_clr_l6 + ba,pt %xcc, rtrap nop /* Be very careful about usage of the trap globals here. @@ -100,7 +100,7 @@ spill_fixup_dax: rd %pc, %g7 call do_sparc64_fault add %sp, PTREGS_OFF, %o0 - ba,a,pt %xcc, rtrap_clr_l6 + ba,a,pt %xcc, rtrap winfix_mna: andn %g3, 0x7f, %g3 @@ -122,12 +122,12 @@ fill_fixup_mna: mov %l4, %o2 call sun4v_do_mna mov %l5, %o1 - ba,a,pt %xcc, rtrap_clr_l6 + ba,a,pt %xcc, rtrap 1: mov %l4, %o1 mov %l5, %o2 call mem_address_unaligned nop - ba,a,pt %xcc, rtrap_clr_l6 + ba,a,pt %xcc, rtrap winfix_dax: andn %g3, 0x7f, %g3 @@ -150,7 +150,7 @@ fill_fixup_dax: add %sp, PTREGS_OFF, %o0 call sun4v_data_access_exception nop - ba,a,pt %xcc, rtrap_clr_l6 + ba,a,pt %xcc, rtrap 1: call spitfire_data_access_exception nop - ba,a,pt %xcc, rtrap_clr_l6 + ba,a,pt %xcc, rtrap diff --git a/arch/sparc64/lib/iomap.c b/arch/sparc64/lib/iomap.c index ac556db0697..7120ebbd4d0 100644 --- a/arch/sparc64/lib/iomap.c +++ b/arch/sparc64/lib/iomap.c @@ -21,8 +21,8 @@ EXPORT_SYMBOL(ioport_unmap); /* Create a virtual mapping cookie for a PCI BAR (memory or IO) */ void __iomem *pci_iomap(struct pci_dev *dev, int bar, unsigned long maxlen) { - unsigned long start = pci_resource_start(dev, bar); - unsigned long len = pci_resource_len(dev, bar); + resource_size_t start = pci_resource_start(dev, bar); + resource_size_t len = pci_resource_len(dev, bar); unsigned long flags = pci_resource_flags(dev, bar); if (!len || !start) diff --git a/arch/sparc64/mm/init.c b/arch/sparc64/mm/init.c index f37078d9640..4cad0b32b0a 100644 --- a/arch/sparc64/mm/init.c +++ b/arch/sparc64/mm/init.c @@ -24,6 +24,8 @@ #include <linux/cache.h> #include <linux/sort.h> #include <linux/percpu.h> +#include <linux/lmb.h> +#include <linux/mmzone.h> #include <asm/head.h> #include <asm/system.h> @@ -72,9 +74,7 @@ extern struct tsb swapper_4m_tsb[KERNEL_TSB4M_NENTRIES]; #define MAX_BANKS 32 static struct linux_prom64_registers pavail[MAX_BANKS] __initdata; -static struct linux_prom64_registers pavail_rescan[MAX_BANKS] __initdata; static int pavail_ents __initdata; -static int pavail_rescan_ents __initdata; static int cmp_p64(const void *a, const void *b) { @@ -160,6 +160,7 @@ extern unsigned int sparc_ramdisk_image; extern unsigned int sparc_ramdisk_size; struct page *mem_map_zero __read_mostly; +EXPORT_SYMBOL(mem_map_zero); unsigned int sparc64_highest_unlocked_tlb_ent __read_mostly; @@ -715,286 +716,685 @@ out: smp_new_mmu_context_version(); } -/* Find a free area for the bootmem map, avoiding the kernel image - * and the initial ramdisk. - */ -static unsigned long __init choose_bootmap_pfn(unsigned long start_pfn, - unsigned long end_pfn) +static int numa_enabled = 1; +static int numa_debug; + +static int __init early_numa(char *p) { - unsigned long avoid_start, avoid_end, bootmap_size; - int i; + if (!p) + return 0; + + if (strstr(p, "off")) + numa_enabled = 0; - bootmap_size = bootmem_bootmap_pages(end_pfn - start_pfn); - bootmap_size <<= PAGE_SHIFT; + if (strstr(p, "debug")) + numa_debug = 1; - avoid_start = avoid_end = 0; + return 0; +} +early_param("numa", early_numa); + +#define numadbg(f, a...) \ +do { if (numa_debug) \ + printk(KERN_INFO f, ## a); \ +} while (0) + +static void __init find_ramdisk(unsigned long phys_base) +{ #ifdef CONFIG_BLK_DEV_INITRD - avoid_start = initrd_start; - avoid_end = PAGE_ALIGN(initrd_end); + if (sparc_ramdisk_image || sparc_ramdisk_image64) { + unsigned long ramdisk_image; + + /* Older versions of the bootloader only supported a + * 32-bit physical address for the ramdisk image + * location, stored at sparc_ramdisk_image. Newer + * SILO versions set sparc_ramdisk_image to zero and + * provide a full 64-bit physical address at + * sparc_ramdisk_image64. + */ + ramdisk_image = sparc_ramdisk_image; + if (!ramdisk_image) + ramdisk_image = sparc_ramdisk_image64; + + /* Another bootloader quirk. The bootloader normalizes + * the physical address to KERNBASE, so we have to + * factor that back out and add in the lowest valid + * physical page address to get the true physical address. + */ + ramdisk_image -= KERNBASE; + ramdisk_image += phys_base; + + numadbg("Found ramdisk at physical address 0x%lx, size %u\n", + ramdisk_image, sparc_ramdisk_size); + + initrd_start = ramdisk_image; + initrd_end = ramdisk_image + sparc_ramdisk_size; + + lmb_reserve(initrd_start, initrd_end); + } #endif +} - for (i = 0; i < pavail_ents; i++) { - unsigned long start, end; +struct node_mem_mask { + unsigned long mask; + unsigned long val; + unsigned long bootmem_paddr; +}; +static struct node_mem_mask node_masks[MAX_NUMNODES]; +static int num_node_masks; - start = pavail[i].phys_addr; - end = start + pavail[i].reg_size; +int numa_cpu_lookup_table[NR_CPUS]; +cpumask_t numa_cpumask_lookup_table[MAX_NUMNODES]; - while (start < end) { - if (start >= kern_base && - start < PAGE_ALIGN(kern_base + kern_size)) { - start = PAGE_ALIGN(kern_base + kern_size); - continue; - } - if (start >= avoid_start && start < avoid_end) { - start = avoid_end; - continue; - } +#ifdef CONFIG_NEED_MULTIPLE_NODES +static bootmem_data_t plat_node_bdata[MAX_NUMNODES]; - if ((end - start) < bootmap_size) - break; +struct mdesc_mblock { + u64 base; + u64 size; + u64 offset; /* RA-to-PA */ +}; +static struct mdesc_mblock *mblocks; +static int num_mblocks; - if (start < kern_base && - (start + bootmap_size) > kern_base) { - start = PAGE_ALIGN(kern_base + kern_size); - continue; - } +static unsigned long ra_to_pa(unsigned long addr) +{ + int i; - if (start < avoid_start && - (start + bootmap_size) > avoid_start) { - start = avoid_end; - continue; - } + for (i = 0; i < num_mblocks; i++) { + struct mdesc_mblock *m = &mblocks[i]; - /* OK, it doesn't overlap anything, use it. */ - return start >> PAGE_SHIFT; + if (addr >= m->base && + addr < (m->base + m->size)) { + addr += m->offset; + break; } } - - prom_printf("Cannot find free area for bootmap, aborting.\n"); - prom_halt(); + return addr; } -static void __init trim_pavail(unsigned long *cur_size_p, - unsigned long *end_of_phys_p) +static int find_node(unsigned long addr) { - unsigned long to_trim = *cur_size_p - cmdline_memory_size; - unsigned long avoid_start, avoid_end; int i; - to_trim = PAGE_ALIGN(to_trim); + addr = ra_to_pa(addr); + for (i = 0; i < num_node_masks; i++) { + struct node_mem_mask *p = &node_masks[i]; - avoid_start = avoid_end = 0; -#ifdef CONFIG_BLK_DEV_INITRD - avoid_start = initrd_start; - avoid_end = PAGE_ALIGN(initrd_end); + if ((addr & p->mask) == p->val) + return i; + } + return -1; +} + +static unsigned long nid_range(unsigned long start, unsigned long end, + int *nid) +{ + *nid = find_node(start); + start += PAGE_SIZE; + while (start < end) { + int n = find_node(start); + + if (n != *nid) + break; + start += PAGE_SIZE; + } + + return start; +} +#else +static unsigned long nid_range(unsigned long start, unsigned long end, + int *nid) +{ + *nid = 0; + return end; +} #endif - /* Trim some pavail[] entries in order to satisfy the - * requested "mem=xxx" kernel command line specification. - * - * We must not trim off the kernel image area nor the - * initial ramdisk range (if any). Also, we must not trim - * any pavail[] entry down to zero in order to preserve - * the invariant that all pavail[] entries have a non-zero - * size which is assumed by all of the code in here. - */ - for (i = 0; i < pavail_ents; i++) { - unsigned long start, end, kern_end; - unsigned long trim_low, trim_high, n; +/* This must be invoked after performing all of the necessary + * add_active_range() calls for 'nid'. We need to be able to get + * correct data from get_pfn_range_for_nid(). + */ +static void __init allocate_node_data(int nid) +{ + unsigned long paddr, num_pages, start_pfn, end_pfn; + struct pglist_data *p; + +#ifdef CONFIG_NEED_MULTIPLE_NODES + paddr = lmb_alloc_nid(sizeof(struct pglist_data), + SMP_CACHE_BYTES, nid, nid_range); + if (!paddr) { + prom_printf("Cannot allocate pglist_data for nid[%d]\n", nid); + prom_halt(); + } + NODE_DATA(nid) = __va(paddr); + memset(NODE_DATA(nid), 0, sizeof(struct pglist_data)); - kern_end = PAGE_ALIGN(kern_base + kern_size); + NODE_DATA(nid)->bdata = &plat_node_bdata[nid]; +#endif - trim_low = start = pavail[i].phys_addr; - trim_high = end = start + pavail[i].reg_size; + p = NODE_DATA(nid); - if (kern_base >= start && - kern_base < end) { - trim_low = kern_base; - if (kern_end >= end) - continue; - } - if (kern_end >= start && - kern_end < end) { - trim_high = kern_end; - } - if (avoid_start && - avoid_start >= start && - avoid_start < end) { - if (trim_low > avoid_start) - trim_low = avoid_start; - if (avoid_end >= end) - continue; - } - if (avoid_end && - avoid_end >= start && - avoid_end < end) { - if (trim_high < avoid_end) - trim_high = avoid_end; + get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); + p->node_start_pfn = start_pfn; + p->node_spanned_pages = end_pfn - start_pfn; + + if (p->node_spanned_pages) { + num_pages = bootmem_bootmap_pages(p->node_spanned_pages); + + paddr = lmb_alloc_nid(num_pages << PAGE_SHIFT, PAGE_SIZE, nid, + nid_range); + if (!paddr) { + prom_printf("Cannot allocate bootmap for nid[%d]\n", + nid); + prom_halt(); } + node_masks[nid].bootmem_paddr = paddr; + } +} + +static void init_node_masks_nonnuma(void) +{ + int i; + + numadbg("Initializing tables for non-numa.\n"); + + node_masks[0].mask = node_masks[0].val = 0; + num_node_masks = 1; + + for (i = 0; i < NR_CPUS; i++) + numa_cpu_lookup_table[i] = 0; - if (trim_high <= trim_low) + numa_cpumask_lookup_table[0] = CPU_MASK_ALL; +} + +#ifdef CONFIG_NEED_MULTIPLE_NODES +struct pglist_data *node_data[MAX_NUMNODES]; + +EXPORT_SYMBOL(numa_cpu_lookup_table); +EXPORT_SYMBOL(numa_cpumask_lookup_table); +EXPORT_SYMBOL(node_data); + +struct mdesc_mlgroup { + u64 node; + u64 latency; + u64 match; + u64 mask; +}; +static struct mdesc_mlgroup *mlgroups; +static int num_mlgroups; + +static int scan_pio_for_cfg_handle(struct mdesc_handle *md, u64 pio, + u32 cfg_handle) +{ + u64 arc; + + mdesc_for_each_arc(arc, md, pio, MDESC_ARC_TYPE_FWD) { + u64 target = mdesc_arc_target(md, arc); + const u64 *val; + + val = mdesc_get_property(md, target, + "cfg-handle", NULL); + if (val && *val == cfg_handle) + return 0; + } + return -ENODEV; +} + +static int scan_arcs_for_cfg_handle(struct mdesc_handle *md, u64 grp, + u32 cfg_handle) +{ + u64 arc, candidate, best_latency = ~(u64)0; + + candidate = MDESC_NODE_NULL; + mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_FWD) { + u64 target = mdesc_arc_target(md, arc); + const char *name = mdesc_node_name(md, target); + const u64 *val; + + if (strcmp(name, "pio-latency-group")) continue; - if (trim_low == start && trim_high == end) { - /* Whole chunk is available for trimming. - * Trim all except one page, in order to keep - * entry non-empty. - */ - n = (end - start) - PAGE_SIZE; - if (n > to_trim) - n = to_trim; - - if (n) { - pavail[i].phys_addr += n; - pavail[i].reg_size -= n; - to_trim -= n; - } - } else { - n = (trim_low - start); - if (n > to_trim) - n = to_trim; - - if (n) { - pavail[i].phys_addr += n; - pavail[i].reg_size -= n; - to_trim -= n; - } - if (to_trim) { - n = end - trim_high; - if (n > to_trim) - n = to_trim; - if (n) { - pavail[i].reg_size -= n; - to_trim -= n; - } - } + val = mdesc_get_property(md, target, "latency", NULL); + if (!val) + continue; + + if (*val < best_latency) { + candidate = target; + best_latency = *val; } + } + + if (candidate == MDESC_NODE_NULL) + return -ENODEV; + + return scan_pio_for_cfg_handle(md, candidate, cfg_handle); +} - if (!to_trim) +int of_node_to_nid(struct device_node *dp) +{ + const struct linux_prom64_registers *regs; + struct mdesc_handle *md; + u32 cfg_handle; + int count, nid; + u64 grp; + + if (!mlgroups) + return -1; + + regs = of_get_property(dp, "reg", NULL); + if (!regs) + return -1; + + cfg_handle = (regs->phys_addr >> 32UL) & 0x0fffffff; + + md = mdesc_grab(); + + count = 0; + nid = -1; + mdesc_for_each_node_by_name(md, grp, "group") { + if (!scan_arcs_for_cfg_handle(md, grp, cfg_handle)) { + nid = count; break; + } + count++; } - /* Recalculate. */ - *cur_size_p = 0UL; - for (i = 0; i < pavail_ents; i++) { - *end_of_phys_p = pavail[i].phys_addr + - pavail[i].reg_size; - *cur_size_p += pavail[i].reg_size; - } + mdesc_release(md); + + return nid; } -/* About pages_avail, this is the value we will use to calculate - * the zholes_size[] argument given to free_area_init_node(). The - * page allocator uses this to calculate nr_kernel_pages, - * nr_all_pages and zone->present_pages. On NUMA it is used - * to calculate zone->min_unmapped_pages and zone->min_slab_pages. - * - * So this number should really be set to what the page allocator - * actually ends up with. This means: - * 1) It should include bootmem map pages, we'll release those. - * 2) It should not include the kernel image, except for the - * __init sections which we will also release. - * 3) It should include the initrd image, since we'll release - * that too. - */ -static unsigned long __init bootmem_init(unsigned long *pages_avail, - unsigned long phys_base) +static void add_node_ranges(void) { - unsigned long bootmap_size, end_pfn; - unsigned long end_of_phys_memory = 0UL; - unsigned long bootmap_pfn, bytes_avail, size; int i; - bytes_avail = 0UL; - for (i = 0; i < pavail_ents; i++) { - end_of_phys_memory = pavail[i].phys_addr + - pavail[i].reg_size; - bytes_avail += pavail[i].reg_size; + for (i = 0; i < lmb.memory.cnt; i++) { + unsigned long size = lmb_size_bytes(&lmb.memory, i); + unsigned long start, end; + + start = lmb.memory.region[i].base; + end = start + size; + while (start < end) { + unsigned long this_end; + int nid; + + this_end = nid_range(start, end, &nid); + + numadbg("Adding active range nid[%d] " + "start[%lx] end[%lx]\n", + nid, start, this_end); + + add_active_range(nid, + start >> PAGE_SHIFT, + this_end >> PAGE_SHIFT); + + start = this_end; + } } +} - /* Determine the location of the initial ramdisk before trying - * to honor the "mem=xxx" command line argument. We must know - * where the kernel image and the ramdisk image are so that we - * do not trim those two areas from the physical memory map. - */ +static int __init grab_mlgroups(struct mdesc_handle *md) +{ + unsigned long paddr; + int count = 0; + u64 node; + + mdesc_for_each_node_by_name(md, node, "memory-latency-group") + count++; + if (!count) + return -ENOENT; + + paddr = lmb_alloc(count * sizeof(struct mdesc_mlgroup), + SMP_CACHE_BYTES); + if (!paddr) + return -ENOMEM; + + mlgroups = __va(paddr); + num_mlgroups = count; + + count = 0; + mdesc_for_each_node_by_name(md, node, "memory-latency-group") { + struct mdesc_mlgroup *m = &mlgroups[count++]; + const u64 *val; + + m->node = node; + + val = mdesc_get_property(md, node, "latency", NULL); + m->latency = *val; + val = mdesc_get_property(md, node, "address-match", NULL); + m->match = *val; + val = mdesc_get_property(md, node, "address-mask", NULL); + m->mask = *val; + + numadbg("MLGROUP[%d]: node[%lx] latency[%lx] " + "match[%lx] mask[%lx]\n", + count - 1, m->node, m->latency, m->match, m->mask); + } -#ifdef CONFIG_BLK_DEV_INITRD - /* Now have to check initial ramdisk, so that bootmap does not overwrite it */ - if (sparc_ramdisk_image || sparc_ramdisk_image64) { - unsigned long ramdisk_image = sparc_ramdisk_image ? - sparc_ramdisk_image : sparc_ramdisk_image64; - ramdisk_image -= KERNBASE; - initrd_start = ramdisk_image + phys_base; - initrd_end = initrd_start + sparc_ramdisk_size; - if (initrd_end > end_of_phys_memory) { - printk(KERN_CRIT "initrd extends beyond end of memory " - "(0x%016lx > 0x%016lx)\ndisabling initrd\n", - initrd_end, end_of_phys_memory); - initrd_start = 0; - initrd_end = 0; + return 0; +} + +static int __init grab_mblocks(struct mdesc_handle *md) +{ + unsigned long paddr; + int count = 0; + u64 node; + + mdesc_for_each_node_by_name(md, node, "mblock") + count++; + if (!count) + return -ENOENT; + + paddr = lmb_alloc(count * sizeof(struct mdesc_mblock), + SMP_CACHE_BYTES); + if (!paddr) + return -ENOMEM; + + mblocks = __va(paddr); + num_mblocks = count; + + count = 0; + mdesc_for_each_node_by_name(md, node, "mblock") { + struct mdesc_mblock *m = &mblocks[count++]; + const u64 *val; + + val = mdesc_get_property(md, node, "base", NULL); + m->base = *val; + val = mdesc_get_property(md, node, "size", NULL); + m->size = *val; + val = mdesc_get_property(md, node, + "address-congruence-offset", NULL); + m->offset = *val; + + numadbg("MBLOCK[%d]: base[%lx] size[%lx] offset[%lx]\n", + count - 1, m->base, m->size, m->offset); + } + + return 0; +} + +static void __init numa_parse_mdesc_group_cpus(struct mdesc_handle *md, + u64 grp, cpumask_t *mask) +{ + u64 arc; + + cpus_clear(*mask); + + mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_BACK) { + u64 target = mdesc_arc_target(md, arc); + const char *name = mdesc_node_name(md, target); + const u64 *id; + + if (strcmp(name, "cpu")) + continue; + id = mdesc_get_property(md, target, "id", NULL); + if (*id < NR_CPUS) + cpu_set(*id, *mask); + } +} + +static struct mdesc_mlgroup * __init find_mlgroup(u64 node) +{ + int i; + + for (i = 0; i < num_mlgroups; i++) { + struct mdesc_mlgroup *m = &mlgroups[i]; + if (m->node == node) + return m; + } + return NULL; +} + +static int __init numa_attach_mlgroup(struct mdesc_handle *md, u64 grp, + int index) +{ + struct mdesc_mlgroup *candidate = NULL; + u64 arc, best_latency = ~(u64)0; + struct node_mem_mask *n; + + mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_FWD) { + u64 target = mdesc_arc_target(md, arc); + struct mdesc_mlgroup *m = find_mlgroup(target); + if (!m) + continue; + if (m->latency < best_latency) { + candidate = m; + best_latency = m->latency; } } -#endif + if (!candidate) + return -ENOENT; + + if (num_node_masks != index) { + printk(KERN_ERR "Inconsistent NUMA state, " + "index[%d] != num_node_masks[%d]\n", + index, num_node_masks); + return -EINVAL; + } - if (cmdline_memory_size && - bytes_avail > cmdline_memory_size) - trim_pavail(&bytes_avail, - &end_of_phys_memory); + n = &node_masks[num_node_masks++]; - *pages_avail = bytes_avail >> PAGE_SHIFT; + n->mask = candidate->mask; + n->val = candidate->match; - end_pfn = end_of_phys_memory >> PAGE_SHIFT; + numadbg("NUMA NODE[%d]: mask[%lx] val[%lx] (latency[%lx])\n", + index, n->mask, n->val, candidate->latency); - /* Initialize the boot-time allocator. */ - max_pfn = max_low_pfn = end_pfn; - min_low_pfn = (phys_base >> PAGE_SHIFT); + return 0; +} - bootmap_pfn = choose_bootmap_pfn(min_low_pfn, end_pfn); +static int __init numa_parse_mdesc_group(struct mdesc_handle *md, u64 grp, + int index) +{ + cpumask_t mask; + int cpu; - bootmap_size = init_bootmem_node(NODE_DATA(0), bootmap_pfn, - min_low_pfn, end_pfn); + numa_parse_mdesc_group_cpus(md, grp, &mask); - /* Now register the available physical memory with the - * allocator. - */ - for (i = 0; i < pavail_ents; i++) - free_bootmem(pavail[i].phys_addr, pavail[i].reg_size); + for_each_cpu_mask(cpu, mask) + numa_cpu_lookup_table[cpu] = index; + numa_cpumask_lookup_table[index] = mask; -#ifdef CONFIG_BLK_DEV_INITRD - if (initrd_start) { - size = initrd_end - initrd_start; + if (numa_debug) { + printk(KERN_INFO "NUMA GROUP[%d]: cpus [ ", index); + for_each_cpu_mask(cpu, mask) + printk("%d ", cpu); + printk("]\n"); + } - /* Reserve the initrd image area. */ - reserve_bootmem(initrd_start, size, BOOTMEM_DEFAULT); + return numa_attach_mlgroup(md, grp, index); +} - initrd_start += PAGE_OFFSET; - initrd_end += PAGE_OFFSET; +static int __init numa_parse_mdesc(void) +{ + struct mdesc_handle *md = mdesc_grab(); + int i, err, count; + u64 node; + + node = mdesc_node_by_name(md, MDESC_NODE_NULL, "latency-groups"); + if (node == MDESC_NODE_NULL) { + mdesc_release(md); + return -ENOENT; + } + + err = grab_mblocks(md); + if (err < 0) + goto out; + + err = grab_mlgroups(md); + if (err < 0) + goto out; + + count = 0; + mdesc_for_each_node_by_name(md, node, "group") { + err = numa_parse_mdesc_group(md, node, count); + if (err < 0) + break; + count++; } + + add_node_ranges(); + + for (i = 0; i < num_node_masks; i++) { + allocate_node_data(i); + node_set_online(i); + } + + err = 0; +out: + mdesc_release(md); + return err; +} + +static int __init numa_parse_sun4u(void) +{ + return -1; +} + +static int __init bootmem_init_numa(void) +{ + int err = -1; + + numadbg("bootmem_init_numa()\n"); + + if (numa_enabled) { + if (tlb_type == hypervisor) + err = numa_parse_mdesc(); + else + err = numa_parse_sun4u(); + } + return err; +} + +#else + +static int bootmem_init_numa(void) +{ + return -1; +} + #endif - /* Reserve the kernel text/data/bss. */ - reserve_bootmem(kern_base, kern_size, BOOTMEM_DEFAULT); - *pages_avail -= PAGE_ALIGN(kern_size) >> PAGE_SHIFT; - - /* Add back in the initmem pages. */ - size = ((unsigned long)(__init_end) & PAGE_MASK) - - PAGE_ALIGN((unsigned long)__init_begin); - *pages_avail += size >> PAGE_SHIFT; - - /* Reserve the bootmem map. We do not account for it - * in pages_avail because we will release that memory - * in free_all_bootmem. - */ - size = bootmap_size; - reserve_bootmem((bootmap_pfn << PAGE_SHIFT), size, BOOTMEM_DEFAULT); - for (i = 0; i < pavail_ents; i++) { +static void __init bootmem_init_nonnuma(void) +{ + unsigned long top_of_ram = lmb_end_of_DRAM(); + unsigned long total_ram = lmb_phys_mem_size(); + unsigned int i; + + numadbg("bootmem_init_nonnuma()\n"); + + printk(KERN_INFO "Top of RAM: 0x%lx, Total RAM: 0x%lx\n", + top_of_ram, total_ram); + printk(KERN_INFO "Memory hole size: %ldMB\n", + (top_of_ram - total_ram) >> 20); + + init_node_masks_nonnuma(); + + for (i = 0; i < lmb.memory.cnt; i++) { + unsigned long size = lmb_size_bytes(&lmb.memory, i); unsigned long start_pfn, end_pfn; - start_pfn = pavail[i].phys_addr >> PAGE_SHIFT; - end_pfn = (start_pfn + (pavail[i].reg_size >> PAGE_SHIFT)); - memory_present(0, start_pfn, end_pfn); + if (!size) + continue; + + start_pfn = lmb.memory.region[i].base >> PAGE_SHIFT; + end_pfn = start_pfn + lmb_size_pages(&lmb.memory, i); + add_active_range(0, start_pfn, end_pfn); } + allocate_node_data(0); + + node_set_online(0); +} + +static void __init reserve_range_in_node(int nid, unsigned long start, + unsigned long end) +{ + numadbg(" reserve_range_in_node(nid[%d],start[%lx],end[%lx]\n", + nid, start, end); + while (start < end) { + unsigned long this_end; + int n; + + this_end = nid_range(start, end, &n); + if (n == nid) { + numadbg(" MATCH reserving range [%lx:%lx]\n", + start, this_end); + reserve_bootmem_node(NODE_DATA(nid), start, + (this_end - start), BOOTMEM_DEFAULT); + } else + numadbg(" NO MATCH, advancing start to %lx\n", + this_end); + + start = this_end; + } +} + +static void __init trim_reserved_in_node(int nid) +{ + int i; + + numadbg(" trim_reserved_in_node(%d)\n", nid); + + for (i = 0; i < lmb.reserved.cnt; i++) { + unsigned long start = lmb.reserved.region[i].base; + unsigned long size = lmb_size_bytes(&lmb.reserved, i); + unsigned long end = start + size; + + reserve_range_in_node(nid, start, end); + } +} + +static void __init bootmem_init_one_node(int nid) +{ + struct pglist_data *p; + + numadbg("bootmem_init_one_node(%d)\n", nid); + + p = NODE_DATA(nid); + + if (p->node_spanned_pages) { + unsigned long paddr = node_masks[nid].bootmem_paddr; + unsigned long end_pfn; + + end_pfn = p->node_start_pfn + p->node_spanned_pages; + + numadbg(" init_bootmem_node(%d, %lx, %lx, %lx)\n", + nid, paddr >> PAGE_SHIFT, p->node_start_pfn, end_pfn); + + init_bootmem_node(p, paddr >> PAGE_SHIFT, + p->node_start_pfn, end_pfn); + + numadbg(" free_bootmem_with_active_regions(%d, %lx)\n", + nid, end_pfn); + free_bootmem_with_active_regions(nid, end_pfn); + + trim_reserved_in_node(nid); + + numadbg(" sparse_memory_present_with_active_regions(%d)\n", + nid); + sparse_memory_present_with_active_regions(nid); + } +} + +static unsigned long __init bootmem_init(unsigned long phys_base) +{ + unsigned long end_pfn; + int nid; + + end_pfn = lmb_end_of_DRAM() >> PAGE_SHIFT; + max_pfn = max_low_pfn = end_pfn; + min_low_pfn = (phys_base >> PAGE_SHIFT); + + if (bootmem_init_numa() < 0) + bootmem_init_nonnuma(); + + /* XXX cpu notifier XXX */ + + for_each_online_node(nid) + bootmem_init_one_node(nid); + sparse_init(); return end_pfn; @@ -1289,7 +1689,7 @@ void __init setup_per_cpu_areas(void) void __init paging_init(void) { - unsigned long end_pfn, pages_avail, shift, phys_base; + unsigned long end_pfn, shift, phys_base; unsigned long real_end, i; /* These build time checkes make sure that the dcache_dirty_cpu() @@ -1300,9 +1700,21 @@ void __init paging_init(void) * functions like clear_dcache_dirty_cpu use the cpu mask * in 13-bit signed-immediate instruction fields. */ - BUILD_BUG_ON(FLAGS_RESERVED != 32); + + /* + * Page flags must not reach into upper 32 bits that are used + * for the cpu number + */ + BUILD_BUG_ON(NR_PAGEFLAGS > 32); + + /* + * The bit fields placed in the high range must not reach below + * the 32 bit boundary. Otherwise we cannot place the cpu field + * at the 32 bit boundary. + */ BUILD_BUG_ON(SECTIONS_WIDTH + NODES_WIDTH + ZONES_WIDTH + - ilog2(roundup_pow_of_two(NR_CPUS)) > FLAGS_RESERVED); + ilog2(roundup_pow_of_two(NR_CPUS)) > 32); + BUILD_BUG_ON(NR_CPUS > 4096); kern_base = (prom_boot_mapping_phys_low >> 22UL) << 22UL; @@ -1330,12 +1742,26 @@ void __init paging_init(void) sun4v_ktsb_init(); } + lmb_init(); + /* Find available physical memory... */ read_obp_memory("available", &pavail[0], &pavail_ents); phys_base = 0xffffffffffffffffUL; - for (i = 0; i < pavail_ents; i++) + for (i = 0; i < pavail_ents; i++) { phys_base = min(phys_base, pavail[i].phys_addr); + lmb_add(pavail[i].phys_addr, pavail[i].reg_size); + } + + lmb_reserve(kern_base, kern_size); + + find_ramdisk(phys_base); + + if (cmdline_memory_size) + lmb_enforce_memory_limit(phys_base + cmdline_memory_size); + + lmb_analyze(); + lmb_dump_all(); set_bit(0, mmu_context_bmap); @@ -1371,14 +1797,10 @@ void __init paging_init(void) if (tlb_type == hypervisor) sun4v_ktsb_register(); - /* Setup bootmem... */ - pages_avail = 0; - last_valid_pfn = end_pfn = bootmem_init(&pages_avail, phys_base); - - max_mapnr = last_valid_pfn; - - kernel_physical_mapping_init(); - + /* We must setup the per-cpu areas before we pull in the + * PROM and the MDESC. The code there fills in cpu and + * other information into per-cpu data structures. + */ real_setup_per_cpu_areas(); prom_build_devicetree(); @@ -1386,20 +1808,22 @@ void __init paging_init(void) if (tlb_type == hypervisor) sun4v_mdesc_init(); + /* Setup bootmem... */ + last_valid_pfn = end_pfn = bootmem_init(phys_base); + +#ifndef CONFIG_NEED_MULTIPLE_NODES + max_mapnr = last_valid_pfn; +#endif + kernel_physical_mapping_init(); + { - unsigned long zones_size[MAX_NR_ZONES]; - unsigned long zholes_size[MAX_NR_ZONES]; - int znum; + unsigned long max_zone_pfns[MAX_NR_ZONES]; - for (znum = 0; znum < MAX_NR_ZONES; znum++) - zones_size[znum] = zholes_size[znum] = 0; + memset(max_zone_pfns, 0, sizeof(max_zone_pfns)); - zones_size[ZONE_NORMAL] = end_pfn; - zholes_size[ZONE_NORMAL] = end_pfn - pages_avail; + max_zone_pfns[ZONE_NORMAL] = end_pfn; - free_area_init_node(0, &contig_page_data, zones_size, - __pa(PAGE_OFFSET) >> PAGE_SHIFT, - zholes_size); + free_area_init_nodes(max_zone_pfns); } printk("Booting Linux...\n"); @@ -1408,21 +1832,52 @@ void __init paging_init(void) cpu_probe(); } -static void __init taint_real_pages(void) +int __init page_in_phys_avail(unsigned long paddr) +{ + int i; + + paddr &= PAGE_MASK; + + for (i = 0; i < pavail_ents; i++) { + unsigned long start, end; + + start = pavail[i].phys_addr; + end = start + pavail[i].reg_size; + + if (paddr >= start && paddr < end) + return 1; + } + if (paddr >= kern_base && paddr < (kern_base + kern_size)) + return 1; +#ifdef CONFIG_BLK_DEV_INITRD + if (paddr >= __pa(initrd_start) && + paddr < __pa(PAGE_ALIGN(initrd_end))) + return 1; +#endif + + return 0; +} + +static struct linux_prom64_registers pavail_rescan[MAX_BANKS] __initdata; +static int pavail_rescan_ents __initdata; + +/* Certain OBP calls, such as fetching "available" properties, can + * claim physical memory. So, along with initializing the valid + * address bitmap, what we do here is refetch the physical available + * memory list again, and make sure it provides at least as much + * memory as 'pavail' does. + */ +static void setup_valid_addr_bitmap_from_pavail(void) { int i; read_obp_memory("available", &pavail_rescan[0], &pavail_rescan_ents); - /* Find changes discovered in the physmem available rescan and - * reserve the lost portions in the bootmem maps. - */ for (i = 0; i < pavail_ents; i++) { unsigned long old_start, old_end; old_start = pavail[i].phys_addr; - old_end = old_start + - pavail[i].reg_size; + old_end = old_start + pavail[i].reg_size; while (old_start < old_end) { int n; @@ -1440,7 +1895,16 @@ static void __init taint_real_pages(void) goto do_next_page; } } - reserve_bootmem(old_start, PAGE_SIZE, BOOTMEM_DEFAULT); + + prom_printf("mem_init: Lost memory in pavail\n"); + prom_printf("mem_init: OLD start[%lx] size[%lx]\n", + pavail[i].phys_addr, + pavail[i].reg_size); + prom_printf("mem_init: NEW start[%lx] size[%lx]\n", + pavail_rescan[i].phys_addr, + pavail_rescan[i].reg_size); + prom_printf("mem_init: Cannot continue, aborting.\n"); + prom_halt(); do_next_page: old_start += PAGE_SIZE; @@ -1448,32 +1912,6 @@ static void __init taint_real_pages(void) } } -int __init page_in_phys_avail(unsigned long paddr) -{ - int i; - - paddr &= PAGE_MASK; - - for (i = 0; i < pavail_rescan_ents; i++) { - unsigned long start, end; - - start = pavail_rescan[i].phys_addr; - end = start + pavail_rescan[i].reg_size; - - if (paddr >= start && paddr < end) - return 1; - } - if (paddr >= kern_base && paddr < (kern_base + kern_size)) - return 1; -#ifdef CONFIG_BLK_DEV_INITRD - if (paddr >= __pa(initrd_start) && - paddr < __pa(PAGE_ALIGN(initrd_end))) - return 1; -#endif - - return 0; -} - void __init mem_init(void) { unsigned long codepages, datapages, initpages; @@ -1496,14 +1934,26 @@ void __init mem_init(void) addr += PAGE_SIZE; } - taint_real_pages(); + setup_valid_addr_bitmap_from_pavail(); high_memory = __va(last_valid_pfn << PAGE_SHIFT); +#ifdef CONFIG_NEED_MULTIPLE_NODES + for_each_online_node(i) { + if (NODE_DATA(i)->node_spanned_pages != 0) { + totalram_pages += + free_all_bootmem_node(NODE_DATA(i)); + } + } +#else + totalram_pages = free_all_bootmem(); +#endif + /* We subtract one to account for the mem_map_zero page * allocated below. */ - totalram_pages = num_physpages = free_all_bootmem() - 1; + totalram_pages -= 1; + num_physpages = totalram_pages; /* * Set up the zero page, mark it reserved, so that page count diff --git a/arch/sparc64/mm/tsb.c b/arch/sparc64/mm/tsb.c index a3e6e4b635b..fe70c8a557b 100644 --- a/arch/sparc64/mm/tsb.c +++ b/arch/sparc64/mm/tsb.c @@ -321,7 +321,8 @@ retry_tsb_alloc: if (new_size > (PAGE_SIZE * 2)) gfp_flags = __GFP_NOWARN | __GFP_NORETRY; - new_tsb = kmem_cache_alloc(tsb_caches[new_cache_index], gfp_flags); + new_tsb = kmem_cache_alloc_node(tsb_caches[new_cache_index], + gfp_flags, numa_node_id()); if (unlikely(!new_tsb)) { /* Not being able to fork due to a high-order TSB * allocation failure is very bad behavior. Just back diff --git a/arch/sparc64/mm/ultra.S b/arch/sparc64/mm/ultra.S index 2865c105b6a..796e005dad8 100644 --- a/arch/sparc64/mm/ultra.S +++ b/arch/sparc64/mm/ultra.S @@ -476,7 +476,6 @@ xcall_sync_tick: #endif call smp_synchronize_tick_client nop - clr %l6 b rtrap_xcall ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1 @@ -511,7 +510,6 @@ xcall_report_regs: #endif call __show_regs add %sp, PTREGS_OFF, %o0 - clr %l6 /* Has to be a non-v9 branch due to the large distance. */ b rtrap_xcall ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1 @@ -576,7 +574,7 @@ __hypervisor_tlb_xcall_error: mov %l4, %o0 call hypervisor_tlbop_error_xcall mov %l5, %o1 - ba,a,pt %xcc, rtrap_clr_l6 + ba,a,pt %xcc, rtrap .globl __hypervisor_xcall_flush_tlb_mm __hypervisor_xcall_flush_tlb_mm: /* 21 insns */ @@ -678,6 +676,33 @@ xcall_new_mmu_context_version: wr %g0, (1 << PIL_SMP_CTX_NEW_VERSION), %set_softint retry +#ifdef CONFIG_KGDB + .globl xcall_kgdb_capture +xcall_kgdb_capture: +661: rdpr %pstate, %g2 + wrpr %g2, PSTATE_IG | PSTATE_AG, %pstate + .section .sun4v_2insn_patch, "ax" + .word 661b + nop + nop + .previous + + rdpr %pil, %g2 + wrpr %g0, 15, %pil + sethi %hi(109f), %g7 + ba,pt %xcc, etrap_irq +109: or %g7, %lo(109b), %g7 +#ifdef CONFIG_TRACE_IRQFLAGS + call trace_hardirqs_off + nop +#endif + call smp_kgdb_capture_client + add %sp, PTREGS_OFF, %o0 + /* Has to be a non-v9 branch due to the large distance. */ + ba rtrap_xcall + ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1 +#endif + #endif /* CONFIG_SMP */ |