perf_event: aml pmu interrupt issue fixup

PD#167574: perf_event: aml pmu interrupt issue fixup

amlogic arm pmu have a issue that all core's interrupts routes to
one gic SPI interrupt,
when some core raise a pmu interrupt(arm pmu counter overflow),
the global gic SPI interrupt will raise(default on cpu0),
and we can obtain core info which caused interrupt from
sys_cpu_status0 reg.

In global pmu interrupt handler we distinguish interrupts from other cpu,
then send a AML ipi interrupt and wait that cpu complete pmu interrupt.

Change-Id: I28ada689e5b94671c8cfb6189e46134c3c6804cd
Signed-off-by: Hanjie Lin <hanjie.lin@amlogic.com>

arch/arm64/boot/dts/amlogic/mesonaxg.dtsi

@@ -121,12 +121,22 @@
 	};
 	arm_pmu {
 		compatible = "arm,armv8-pmuv3";
-		interrupts = <0 137 4>,
-			     <0 138 4>,
-			     <0 153 4>,
-			     <0 154 4>;
+		interrupts = <0 137 4>;
+		reg = <0x0 0xff634400 0 0x1000>;
+
+		/* addr = base + offset << 2 */
+		sys_cpu_status0_offset = <0xa0>;
+
+		sys_cpu_status0_pmuirq_mask = <0xf>;
+
+		/* default 10ms */
+		relax_timer_ns = <10000000>;
+
+		/* default 10000us */
+		max_wait_cnt = <10000>;
 	};
 
+
 	gic: interrupt-controller@2c001000 {
 		compatible = "arm,cortex-a15-gic", "arm,cortex-a9-gic";
 		#interrupt-cells = <3>;

arch/arm64/boot/dts/amlogic/mesong12a.dtsi

@@ -158,6 +158,18 @@
 	arm_pmu {
 		compatible = "arm,armv8-pmuv3";
 		interrupts = <0 137 4>;
+		reg = <0x0 0xff634400 0 0x1000>;
+
+		/* addr = base + offset << 2 */
+		sys_cpu_status0_offset = <0xa0>;
+
+		sys_cpu_status0_pmuirq_mask = <0xf>;
+
+		/* default 10ms */
+		relax_timer_ns = <10000000>;
+
+		/* default 10000us */
+		max_wait_cnt = <10000>;
 	};
 
 	gic: interrupt-controller@2c001000 {

arch/arm64/boot/dts/amlogic/mesongxl.dtsi

@@ -135,10 +135,19 @@
 	};
 	arm_pmu {
 		compatible = "arm,armv8-pmuv3";
-		interrupts = <0 137 4>,
-			     <0 138 4>,
-			     <0 153 4>,
-			     <0 154 4>;
+		interrupts = <0 137 4>;
+		reg = <0x0 0xc8834400 0 0x1000>;
+
+		/* addr = base + offset << 2 */
+		sys_cpu_status0_offset = <0xa0>;
+
+		sys_cpu_status0_pmuirq_mask = <0xf>;
+
+		/* default 10ms */
+		relax_timer_ns = <10000000>;
+
+		/* default 10000us */
+		max_wait_cnt = <10000>;
 	};
 
 	gic: interrupt-controller@2c001000 {

arch/arm64/boot/dts/amlogic/mesontxlx.dtsi

@@ -134,10 +134,19 @@
 
 	arm_pmu {
 		compatible = "arm,armv8-pmuv3";
-		interrupts = <0 137 4>,
-			     <0 138 4>,
-			     <0 153 4>,
-			     <0 154 4>;
+		interrupts = <0 137 4>;
+		reg = <0x0 0xff634400 0 0x1000>;
+
+		/* addr = base + offset << 2 */
+		sys_cpu_status0_offset = <0xa0>;
+
+		sys_cpu_status0_pmuirq_mask = <0xf>;
+
+		/* default 10ms */
+		relax_timer_ns = <10000000>;
+
+		/* default 10000us */
+		max_wait_cnt = <10000>;
 	};
 
 	gic: interrupt-controller@2c001000 {

arch/arm64/include/asm/hardirq.h

@@ -20,7 +20,11 @@
 #include <linux/threads.h>
 #include <asm/irq.h>
 
+#ifdef CONFIG_AMLOGIC_MODIFY
+#define NR_IPI	7
+#else
 #define NR_IPI	6
+#endif
 
 typedef struct {
 	unsigned int __softirq_pending;

arch/arm64/include/asm/perf_event.h

@@ -18,6 +18,9 @@
 #define __ASM_PERF_EVENT_H
 
 #include <asm/stack_pointer.h>
+#ifdef CONFIG_AMLOGIC_MODIFY
+#include <linux/hrtimer.h>
+#endif
 
 #define	ARMV8_PMU_MAX_COUNTERS	32
 #define	ARMV8_PMU_COUNTER_MASK	(ARMV8_PMU_MAX_COUNTERS - 1)
@@ -88,4 +91,51 @@ extern unsigned long perf_misc_flags(struct pt_regs *regs);
 	(regs)->pstate = PSR_MODE_EL1h;	\
 }
 
+
+#ifdef CONFIG_AMLOGIC_MODIFY
+
+extern void armv8pmu_handle_irq_ipi(void);
+
+struct amlpmu_fixup_cpuinfo {
+	int irq_num;
+
+	int fix_done;
+
+	unsigned long irq_cnt;
+	unsigned long empty_irq_cnt;
+
+	unsigned long irq_time;
+	unsigned long empty_irq_time;
+
+	unsigned long last_irq_cnt;
+	unsigned long last_empty_irq_cnt;
+
+	unsigned long last_irq_time;
+	unsigned long last_empty_irq_time;
+};
+
+struct amlpmu_fixup_context {
+	struct amlpmu_fixup_cpuinfo __percpu *cpuinfo;
+
+	/* struct arm_pmu */
+	void *dev;
+
+	/* sys_cpu_status0 reg */
+	unsigned int *sys_cpu_status0;
+
+	/*
+	 * In main pmu irq route wait for other cpu fix done may cause lockup,
+	 * when lockup we disable main irq for a while.
+	 * relax_timer will enable main irq again.
+	 */
+	struct hrtimer relax_timer;
+
+	/* dts prop */
+	unsigned int sys_cpu_status0_offset;
+	unsigned int sys_cpu_status0_pmuirq_mask;
+	unsigned int relax_timer_ns;
+	unsigned int max_wait_cnt;
+};
+#endif
+
 #endif

arch/arm64/kernel/perf_event.c

@@ -19,6 +19,10 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#ifdef CONFIG_AMLOGIC_MODIFY
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#endif
+
 #include <asm/irq_regs.h>
 #include <asm/perf_event.h>
 #include <asm/sysreg.h>
@@ -29,6 +33,19 @@
 #include <linux/perf/arm_pmu.h>
 #include <linux/platform_device.h>
 
+
+#ifdef CONFIG_AMLOGIC_MODIFY
+#include <linux/of.h>
+#include <linux/of_irq.h>
+#include <linux/of_device.h>
+#include <linux/irq.h>
+#include <asm/irq.h>
+#include <linux/interrupt.h>
+#include <linux/irqdesc.h>
+#include <linux/of_address.h>
+#include <linux/delay.h>
+#endif
+
 /*
  * ARMv8 PMUv3 Performance Events handling code.
  * Common event types (some are defined in asm/perf_event.h).
@@ -748,6 +765,165 @@ static void armv8pmu_disable_event(struct perf_event *event)
 	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 }
 
+#ifdef CONFIG_AMLOGIC_MODIFY
+
+static struct amlpmu_fixup_context amlpmu_fixup_ctx;
+
+static enum hrtimer_restart amlpmu_relax_timer_func(struct hrtimer *timer)
+{
+	struct amlpmu_fixup_cpuinfo *ci;
+
+	ci = per_cpu_ptr(amlpmu_fixup_ctx.cpuinfo, 0);
+
+	pr_alert("enable cpu0_irq %d again, irq cnt = %lu\n",
+		ci->irq_num,
+		ci->irq_cnt);
+	enable_irq(ci->irq_num);
+
+	return HRTIMER_NORESTART;
+}
+
+
+static void amlpmu_relax_timer_start(int other_cpu)
+{
+	struct amlpmu_fixup_cpuinfo *ci;
+	int cpu;
+
+	cpu = smp_processor_id();
+	WARN_ON(cpu != 0);
+
+	ci = per_cpu_ptr(amlpmu_fixup_ctx.cpuinfo, 0);
+
+	pr_alert("wait cpu %d fixup done timeout, main cpu irq cnt = %lu\n",
+			other_cpu,
+			ci->irq_cnt);
+
+	if (hrtimer_active(&amlpmu_fixup_ctx.relax_timer)) {
+		pr_alert("relax_timer already active, return!\n");
+		return;
+	}
+
+	disable_irq_nosync(ci->irq_num);
+
+	hrtimer_start(&amlpmu_fixup_ctx.relax_timer,
+		ns_to_ktime(amlpmu_fixup_ctx.relax_timer_ns),
+		HRTIMER_MODE_REL);
+}
+
+static irqreturn_t armv8pmu_handle_irq(int irq_num, void *dev);
+
+void armv8pmu_handle_irq_ipi(void)
+{
+	int cpu = smp_processor_id();
+
+	WARN_ON(cpu == 0);
+	WARN_ON(!amlpmu_fixup_ctx.dev);
+
+	armv8pmu_handle_irq(-1, amlpmu_fixup_ctx.dev);
+}
+
+static int aml_pmu_fix(void)
+{
+	int i;
+	int cpu;
+	int pmuirq_val;
+	struct amlpmu_fixup_cpuinfo *ci;
+
+	int max_wait_cnt = amlpmu_fixup_ctx.max_wait_cnt;
+
+	pmuirq_val = readl(amlpmu_fixup_ctx.sys_cpu_status0);
+	pmuirq_val &= amlpmu_fixup_ctx.sys_cpu_status0_pmuirq_mask;
+
+	for (cpu = 0; cpu < num_possible_cpus(); cpu++) {
+		if (pmuirq_val & (1<<cpu)) {
+			if (cpu == 0) {
+				pr_debug("cpu0 shouldn't fix pmuirq = 0x%x\n",
+					pmuirq_val);
+			} else {
+				pr_debug("fix pmu irq cpu %d, pmuirq = 0x%x\n",
+					cpu,
+					pmuirq_val);
+
+				ci = per_cpu_ptr(amlpmu_fixup_ctx.cpuinfo,
+					cpu);
+
+				ci->fix_done = 0;
+
+				/* aml pmu IPI will set fix_done to 1 */
+				mb();
+
+				smp_send_aml_pmu(cpu);
+
+				for (i = 0; i < max_wait_cnt; i++) {
+					if (READ_ONCE(ci->fix_done))
+						break;
+
+					udelay(1);
+				}
+
+				if (i == amlpmu_fixup_ctx.max_wait_cnt)
+					amlpmu_relax_timer_start(cpu);
+
+				return 0;
+			}
+		}
+	}
+
+	return 1;
+}
+
+static void aml_pmu_fix_stat_account(int is_empty_irq)
+{
+	int freq;
+	unsigned long time = jiffies;
+	struct amlpmu_fixup_cpuinfo *ci;
+
+	ci = this_cpu_ptr(amlpmu_fixup_ctx.cpuinfo);
+
+	ci->irq_cnt++;
+	ci->irq_time = time;
+	if (!ci->last_irq_cnt) {
+		ci->last_irq_cnt = ci->irq_cnt;
+		ci->last_irq_time = ci->irq_time;
+	}
+
+	if (is_empty_irq) {
+		ci->empty_irq_cnt++;
+		ci->empty_irq_time = time;
+		if (!ci->last_empty_irq_cnt) {
+			ci->last_empty_irq_cnt = ci->empty_irq_cnt;
+			ci->last_empty_irq_time = ci->empty_irq_time;
+		}
+	}
+
+	if (time_after(ci->irq_time, ci->last_irq_time + HZ)) {
+		freq = ci->irq_cnt - ci->last_irq_cnt;
+		freq = freq * HZ / (ci->irq_time - ci->last_irq_time);
+		pr_debug("irq_cnt = %lu, irq_last_cnt = %lu, freq = %d\n",
+			ci->irq_cnt,
+			ci->last_irq_cnt,
+			freq);
+
+		ci->last_irq_cnt = ci->irq_cnt;
+		ci->last_irq_time = ci->irq_time;
+	}
+
+	if (is_empty_irq &&
+		time_after(ci->empty_irq_time, ci->last_empty_irq_time + HZ)) {
+
+		freq = ci->empty_irq_cnt - ci->last_empty_irq_cnt;
+		freq *= HZ;
+		freq /= (ci->empty_irq_time - ci->last_empty_irq_time);
+		pr_debug("empty_irq_cnt = %lu, freq = %d\n",
+			ci->empty_irq_cnt,
+			freq);
+
+		ci->last_empty_irq_cnt = ci->empty_irq_cnt;
+		ci->last_empty_irq_time = ci->empty_irq_time;
+	}
+}
+#endif
+
 static irqreturn_t armv8pmu_handle_irq(int irq_num, void *dev)
 {
 	u32 pmovsr;
@@ -757,17 +933,45 @@ static irqreturn_t armv8pmu_handle_irq(int irq_num, void *dev)
 	struct pt_regs *regs;
 	int idx;
 
+#ifdef CONFIG_AMLOGIC_MODIFY
+	int cpu;
+	int is_empty_irq = 0;
+	struct amlpmu_fixup_cpuinfo *ci;
+
+	ci = this_cpu_ptr(amlpmu_fixup_ctx.cpuinfo);
+	ci->irq_num = irq_num;
+	amlpmu_fixup_ctx.dev = dev;
+	cpu = smp_processor_id();
+#endif
+
 	/*
 	 * Get and reset the IRQ flags
 	 */
 	pmovsr = armv8pmu_getreset_flags();
 
+#ifdef CONFIG_AMLOGIC_MODIFY
+	ci->fix_done = 1;
+#endif
 	/*
 	 * Did an overflow occur?
 	 */
+#ifdef CONFIG_AMLOGIC_MODIFY
+	if (!armv8pmu_has_overflowed(pmovsr)) {
+		is_empty_irq = 1;
+
+		if (cpu == 0)
+			is_empty_irq = aml_pmu_fix();
+	}
+
+	aml_pmu_fix_stat_account(is_empty_irq);
+
+	/* txlx have some empty pmu irqs, so return IRQ_HANDLED */
+	if (is_empty_irq)
+		return IRQ_HANDLED;
+#else
 	if (!armv8pmu_has_overflowed(pmovsr))
 		return IRQ_NONE;
-
+#endif
 	/*
 	 * Handle the counter(s) overflow(s)
 	 */
@@ -810,6 +1014,9 @@ static irqreturn_t armv8pmu_handle_irq(int irq_num, void *dev)
 	return IRQ_HANDLED;
 }
 
+
+
+
 static void armv8pmu_start(struct arm_pmu *cpu_pmu)
 {
 	unsigned long flags;
@@ -1100,8 +1307,88 @@ static const struct pmu_probe_info armv8_pmu_probe_table[] = {
 	{ /* sentinel value */ }
 };
 
+
+#ifdef CONFIG_AMLOGIC_MODIFY
+static int amlpmu_fixup_init(struct platform_device *pdev)
+{
+	int ret;
+	void __iomem *base;
+
+	amlpmu_fixup_ctx.cpuinfo = __alloc_percpu(
+		sizeof(struct amlpmu_fixup_cpuinfo), 2 * sizeof(void *));
+	if (!amlpmu_fixup_ctx.cpuinfo) {
+		pr_err("alloc percpu failed\n");
+		return -ENOMEM;
+	}
+
+	base = of_iomap(pdev->dev.of_node, 0);
+	if (IS_ERR(base)) {
+		pr_err("of_iomap() failed, base = %p\n", base);
+		return PTR_ERR(base);
+	}
+
+	ret = of_property_read_u32(pdev->dev.of_node,
+		"sys_cpu_status0_offset",
+		&amlpmu_fixup_ctx.sys_cpu_status0_offset);
+	if (ret) {
+		pr_err("read sys_cpu_status0_offset failed, ret = %d\n", ret);
+		return 1;
+	}
+	pr_debug("sys_cpu_status0_offset = 0x%0x\n",
+		amlpmu_fixup_ctx.sys_cpu_status0_offset);
+
+	ret = of_property_read_u32(pdev->dev.of_node,
+		"sys_cpu_status0_pmuirq_mask",
+		&amlpmu_fixup_ctx.sys_cpu_status0_pmuirq_mask);
+	if (ret) {
+		pr_err("read sys_cpu_status0_pmuirq_mask failed, ret = %d\n",
+			ret);
+		return 1;
+	}
+	pr_debug("sys_cpu_status0_pmuirq_mask = 0x%0x\n",
+		amlpmu_fixup_ctx.sys_cpu_status0_pmuirq_mask);
+
+
+	ret = of_property_read_u32(pdev->dev.of_node,
+		"relax_timer_ns",
+		&amlpmu_fixup_ctx.relax_timer_ns);
+	if (ret) {
+		pr_err("read prop relax_timer_ns failed, ret = %d\n", ret);
+		return 1;
+	}
+	pr_debug("relax_timer_ns = %u\n", amlpmu_fixup_ctx.relax_timer_ns);
+
+
+	ret = of_property_read_u32(pdev->dev.of_node,
+		"max_wait_cnt",
+		&amlpmu_fixup_ctx.max_wait_cnt);
+	if (ret) {
+		pr_err("read prop max_wait_cnt failed, ret = %d\n", ret);
+		return 1;
+	}
+	pr_debug("max_wait_cnt = %u\n", amlpmu_fixup_ctx.max_wait_cnt);
+
+
+	base += (amlpmu_fixup_ctx.sys_cpu_status0_offset << 2);
+	amlpmu_fixup_ctx.sys_cpu_status0 = base;
+	pr_debug("sys_cpu_status0 = %p\n", amlpmu_fixup_ctx.sys_cpu_status0);
+
+
+	hrtimer_init(&amlpmu_fixup_ctx.relax_timer,
+		CLOCK_MONOTONIC,
+		HRTIMER_MODE_REL);
+	amlpmu_fixup_ctx.relax_timer.function = amlpmu_relax_timer_func;
+
+	return 0;
+}
+#endif
+
 static int armv8_pmu_device_probe(struct platform_device *pdev)
 {
+#ifdef CONFIG_AMLOGIC_MODIFY
+	if (amlpmu_fixup_init(pdev))
+		return 1;
+#endif
 	if (acpi_disabled)
 		return arm_pmu_device_probe(pdev, armv8_pmu_of_device_ids,
 					    NULL);

arch/arm64/kernel/smp.c

@@ -55,6 +55,10 @@
 #include <asm/ptrace.h>
 #include <asm/virt.h>
 
+#ifdef CONFIG_AMLOGIC_MODIFY
+#include <asm/perf_event.h>
+#endif
+
 #define CREATE_TRACE_POINTS
 #include <trace/events/ipi.h>
 
@@ -76,7 +80,12 @@ enum ipi_msg_type {
 	IPI_CPU_STOP,
 	IPI_TIMER,
 	IPI_IRQ_WORK,
+#ifdef CONFIG_AMLOGIC_MODIFY
+	IPI_WAKEUP,
+	IPI_AML_PMU
+#else
 	IPI_WAKEUP
+#endif
 };
 
 #ifdef CONFIG_ARM64_VHE
@@ -756,6 +765,9 @@ static const char *ipi_types[NR_IPI] __tracepoint_string = {
 	S(IPI_TIMER, "Timer broadcast interrupts"),
 	S(IPI_IRQ_WORK, "IRQ work interrupts"),
 	S(IPI_WAKEUP, "CPU wake-up interrupts"),
+#ifdef CONFIG_AMLOGIC_MODIFY
+	S(IPI_AML_PMU, "AML pmu cross interrupts"),
+#endif
 };
 
 static void smp_cross_call(const struct cpumask *target, unsigned int ipinr)
@@ -764,6 +776,13 @@ static void smp_cross_call(const struct cpumask *target, unsigned int ipinr)
 	__smp_cross_call(target, ipinr);
 }
 
+#ifdef CONFIG_AMLOGIC_MODIFY
+void smp_send_aml_pmu(int cpu)
+{
+	smp_cross_call(cpumask_of(cpu), IPI_AML_PMU);
+}
+#endif
+
 void show_ipi_list(struct seq_file *p, int prec)
 {
 	unsigned int cpu, i;
@@ -827,6 +846,7 @@ static void ipi_cpu_stop(unsigned int cpu)
 		cpu_relax();
 }
 
+
 /*
  * Main handler for inter-processor interrupts
  */
@@ -881,6 +901,12 @@ void handle_IPI(int ipinr, struct pt_regs *regs)
 		break;
 #endif
 
+#ifdef CONFIG_AMLOGIC_MODIFY
+	case IPI_AML_PMU:
+		armv8pmu_handle_irq_ipi();
+		break;
+#endif
+
 	default:
 		pr_crit("CPU%u: Unknown IPI message 0x%x\n", cpu, ipinr);
 		break;

include/linux/smp.h

@@ -73,6 +73,12 @@ extern void smp_send_stop(void);
  */
 extern void smp_send_reschedule(int cpu);
 
+#ifdef CONFIG_AMLOGIC_MODIFY
+/*
+ * sends a 'aml pmu' event to another CPU:
+ */
+extern void smp_send_aml_pmu(int cpu);
+#endif
 
 /*
  * Prepare machine for booting other CPUs.