From 64711f9a47d4defa90417f5e8db8ed0060bc3275 Mon Sep 17 00:00:00 2001
From: Max Filippov <jcmvbkbc@gmail.com>
Date: Wed, 19 Dec 2018 19:48:37 -0800
Subject: [PATCH] xtensa: implement jump_label support
Use 3-byte 'nop' and 'j' instructions that are always present. Don't let
assembler mark a spot right after patchable 'j' instruction as
unreachable and later put literals or padding bytes there. Add separate
implementations of patch_text for SMP and UP cases, avoiding use of
atomics on UP.
Signed-off-by: Max Filippov <jcmvbkbc@gmail.com>
---
.../core/jump-labels/arch-support.txt | 2 +-
arch/xtensa/Kconfig | 1 +
arch/xtensa/include/asm/jump_label.h | 65 ++++++++++++
arch/xtensa/kernel/Makefile | 1 +
arch/xtensa/kernel/jump_label.c | 99 +++++++++++++++++++
5 files changed, 167 insertions(+), 1 deletion(-)
create mode 100644 arch/xtensa/include/asm/jump_label.h
create mode 100644 arch/xtensa/kernel/jump_label.c
diff --git a/Documentation/features/core/jump-labels/arch-support.txt b/Documentation/features/core/jump-labels/arch-support.txt
index 27cbd63abfd28..60111395f932d 100644
--- a/Documentation/features/core/jump-labels/arch-support.txt
+++ b/Documentation/features/core/jump-labels/arch-support.txt
@@ -29,5 +29,5 @@
| um: | TODO |
| unicore32: | TODO |
| x86: | ok |
- | xtensa: | TODO |
+ | xtensa: | ok |
-----------------------
diff --git a/arch/xtensa/Kconfig b/arch/xtensa/Kconfig
index 5a27a6fd3a1c4..92eb80140e7fb 100644
--- a/arch/xtensa/Kconfig
+++ b/arch/xtensa/Kconfig
@@ -17,6 +17,7 @@ config XTENSA
select GENERIC_PCI_IOMAP
select GENERIC_SCHED_CLOCK
select GENERIC_STRNCPY_FROM_USER if KASAN
+ select HAVE_ARCH_JUMP_LABEL
select HAVE_ARCH_KASAN if MMU
select HAVE_ARCH_TRACEHOOK
select HAVE_DEBUG_KMEMLEAK
diff --git a/arch/xtensa/include/asm/jump_label.h b/arch/xtensa/include/asm/jump_label.h
new file mode 100644
index 0000000000000..c812bf85021c0
--- /dev/null
+++ b/arch/xtensa/include/asm/jump_label.h
@@ -0,0 +1,65 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (C) 2018 Cadence Design Systems Inc. */
+
+#ifndef _ASM_XTENSA_JUMP_LABEL_H
+#define _ASM_XTENSA_JUMP_LABEL_H
+
+#ifndef __ASSEMBLY__
+
+#include <linux/types.h>
+
+#define JUMP_LABEL_NOP_SIZE 3
+
+static __always_inline bool arch_static_branch(struct static_key *key,
+ bool branch)
+{
+ asm_volatile_goto("1:\n\t"
+ "_nop\n\t"
+ ".pushsection __jump_table, \"aw\"\n\t"
+ ".word 1b, %l[l_yes], %c0\n\t"
+ ".popsection\n\t"
+ : : "i" (&((char *)key)[branch]) : : l_yes);
+
+ return false;
+l_yes:
+ return true;
+}
+
+static __always_inline bool arch_static_branch_jump(struct static_key *key,
+ bool branch)
+{
+ /*
+ * Xtensa assembler will mark certain points in the code
+ * as unreachable, so that later assembler or linker relaxation
+ * passes could use them. A spot right after the J instruction
+ * is one such point. Assembler and/or linker may insert padding
+ * or literals here, breaking code flow in case the J instruction
+ * is later replaced with NOP. Put a label right after the J to
+ * make it reachable and wrap both into a no-transform block
+ * to avoid any assembler interference with this.
+ */
+ asm_volatile_goto("1:\n\t"
+ ".begin no-transform\n\t"
+ "_j %l[l_yes]\n\t"
+ "2:\n\t"
+ ".end no-transform\n\t"
+ ".pushsection __jump_table, \"aw\"\n\t"
+ ".word 1b, %l[l_yes], %c0\n\t"
+ ".popsection\n\t"
+ : : "i" (&((char *)key)[branch]) : : l_yes);
+
+ return false;
+l_yes:
+ return true;
+}
+
+typedef u32 jump_label_t;
+
+struct jump_entry {
+ jump_label_t code;
+ jump_label_t target;
+ jump_label_t key;
+};
+
+#endif /* __ASSEMBLY__ */
+#endif
diff --git a/arch/xtensa/kernel/Makefile b/arch/xtensa/kernel/Makefile
index 8dff506caf075..6f629027ac7d9 100644
--- a/arch/xtensa/kernel/Makefile
+++ b/arch/xtensa/kernel/Makefile
@@ -16,6 +16,7 @@ obj-$(CONFIG_SMP) += smp.o mxhead.o
obj-$(CONFIG_XTENSA_VARIANT_HAVE_PERF_EVENTS) += perf_event.o
obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o
obj-$(CONFIG_S32C1I_SELFTEST) += s32c1i_selftest.o
+obj-$(CONFIG_JUMP_LABEL) += jump_label.o
# In the Xtensa architecture, assembly generates literals which must always
# precede the L32R instruction with a relative offset less than 256 kB.
diff --git a/arch/xtensa/kernel/jump_label.c b/arch/xtensa/kernel/jump_label.c
new file mode 100644
index 0000000000000..d108f721c1163
--- /dev/null
+++ b/arch/xtensa/kernel/jump_label.c
@@ -0,0 +1,99 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2018 Cadence Design Systems Inc.
+
+#include <linux/cpu.h>
+#include <linux/jump_label.h>
+#include <linux/kernel.h>
+#include <linux/memory.h>
+#include <linux/stop_machine.h>
+#include <linux/types.h>
+
+#include <asm/cacheflush.h>
+
+#ifdef HAVE_JUMP_LABEL
+
+#define J_OFFSET_MASK 0x0003ffff
+#define J_SIGN_MASK (~(J_OFFSET_MASK >> 1))
+
+#if defined(__XTENSA_EL__)
+#define J_INSN 0x6
+#define NOP_INSN 0x0020f0
+#elif defined(__XTENSA_EB__)
+#define J_INSN 0x60000000
+#define NOP_INSN 0x0f020000
+#else
+#error Unsupported endianness.
+#endif
+
+struct patch {
+ atomic_t cpu_count;
+ unsigned long addr;
+ size_t sz;
+ const void *data;
+};
+
+static void local_patch_text(unsigned long addr, const void *data, size_t sz)
+{
+ memcpy((void *)addr, data, sz);
+ local_flush_icache_range(addr, addr + sz);
+}
+
+static int patch_text_stop_machine(void *data)
+{
+ struct patch *patch = data;
+
+ if (atomic_inc_return(&patch->cpu_count) == 1) {
+ local_patch_text(patch->addr, patch->data, patch->sz);
+ atomic_inc(&patch->cpu_count);
+ } else {
+ while (atomic_read(&patch->cpu_count) <= num_online_cpus())
+ cpu_relax();
+ __invalidate_icache_range(patch->addr, patch->sz);
+ }
+ return 0;
+}
+
+static void patch_text(unsigned long addr, const void *data, size_t sz)
+{
+ if (IS_ENABLED(CONFIG_SMP)) {
+ struct patch patch = {
+ .cpu_count = ATOMIC_INIT(0),
+ .addr = addr,
+ .sz = sz,
+ .data = data,
+ };
+ stop_machine_cpuslocked(patch_text_stop_machine,
+ &patch, NULL);
+ } else {
+ unsigned long flags;
+
+ local_irq_save(flags);
+ local_patch_text(addr, data, sz);
+ local_irq_restore(flags);
+ }
+}
+
+void arch_jump_label_transform(struct jump_entry *e,
+ enum jump_label_type type)
+{
+ u32 d = (jump_entry_target(e) - (jump_entry_code(e) + 4));
+ u32 insn;
+
+ /* Jump only works within 128K of the J instruction. */
+ BUG_ON(!((d & J_SIGN_MASK) == 0 ||
+ (d & J_SIGN_MASK) == J_SIGN_MASK));
+
+ if (type == JUMP_LABEL_JMP) {
+#if defined(__XTENSA_EL__)
+ insn = ((d & J_OFFSET_MASK) << 6) | J_INSN;
+#elif defined(__XTENSA_EB__)
+ insn = ((d & J_OFFSET_MASK) << 8) | J_INSN;
+#endif
+ } else {
+ insn = NOP_INSN;
+ }
+
+ patch_text(jump_entry_code(e), &insn, JUMP_LABEL_NOP_SIZE);
+}
+
+#endif /* HAVE_JUMP_LABEL */
--
GitLab