| diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig |
| index 6d5afe2e6ba3..5d92aef787e3 100644 |
| --- a/arch/arm/Kconfig |
| +++ b/arch/arm/Kconfig |
| @@ -33,6 +33,7 @@ config ARM |
| select ARCH_OPTIONAL_KERNEL_RWX_DEFAULT if CPU_V7 |
| select ARCH_SUPPORTS_ATOMIC_RMW |
| select ARCH_SUPPORTS_HUGETLBFS if ARM_LPAE |
| + select ARCH_SUPPORTS_RT if HAVE_POSIX_CPU_TIMERS_TASK_WORK |
| select ARCH_USE_BUILTIN_BSWAP |
| select ARCH_USE_CMPXCHG_LOCKREF |
| select ARCH_USE_MEMTEST |
| @@ -71,7 +72,7 @@ config ARM |
| select HARDIRQS_SW_RESEND |
| select HAVE_ARCH_AUDITSYSCALL if AEABI && !OABI_COMPAT |
| select HAVE_ARCH_BITREVERSE if (CPU_32v7M || CPU_32v7) && !CPU_32v6 |
| - select HAVE_ARCH_JUMP_LABEL if !XIP_KERNEL && !CPU_ENDIAN_BE32 && MMU |
| + select HAVE_ARCH_JUMP_LABEL if !XIP_KERNEL && !CPU_ENDIAN_BE32 && MMU && !PREEMPT_RT |
| select HAVE_ARCH_KFENCE if MMU && !XIP_KERNEL |
| select HAVE_ARCH_KGDB if !CPU_ENDIAN_BE32 && MMU |
| select HAVE_ARCH_KASAN if MMU && !XIP_KERNEL |
| @@ -94,7 +95,7 @@ config ARM |
| select HAVE_DYNAMIC_FTRACE_WITH_REGS if HAVE_DYNAMIC_FTRACE |
| select HAVE_EFFICIENT_UNALIGNED_ACCESS if (CPU_V6 || CPU_V6K || CPU_V7) && MMU |
| select HAVE_EXIT_THREAD |
| - select HAVE_FAST_GUP if ARM_LPAE |
| + select HAVE_FAST_GUP if ARM_LPAE && !(PREEMPT_RT && HIGHPTE) |
| select HAVE_FTRACE_MCOUNT_RECORD if !XIP_KERNEL |
| select HAVE_FUNCTION_GRAPH_TRACER |
| select HAVE_FUNCTION_TRACER if !XIP_KERNEL |
| @@ -115,6 +116,8 @@ config ARM |
| select HAVE_PERF_EVENTS |
| select HAVE_PERF_REGS |
| select HAVE_PERF_USER_STACK_DUMP |
| + select HAVE_POSIX_CPU_TIMERS_TASK_WORK if !KVM |
| + select HAVE_PREEMPT_LAZY |
| select MMU_GATHER_RCU_TABLE_FREE if SMP && ARM_LPAE |
| select HAVE_REGS_AND_STACK_ACCESS_API |
| select HAVE_RSEQ |
| diff --git a/arch/arm/include/asm/thread_info.h b/arch/arm/include/asm/thread_info.h |
| index 7f092cb55a41..ffcbf8ebed4b 100644 |
| --- a/arch/arm/include/asm/thread_info.h |
| +++ b/arch/arm/include/asm/thread_info.h |
| @@ -62,6 +62,7 @@ struct cpu_context_save { |
| struct thread_info { |
| unsigned long flags; /* low level flags */ |
| int preempt_count; /* 0 => preemptable, <0 => bug */ |
| + int preempt_lazy_count; /* 0 => preemptable, <0 => bug */ |
| __u32 cpu; /* cpu */ |
| __u32 cpu_domain; /* cpu domain */ |
| struct cpu_context_save cpu_context; /* cpu context */ |
| @@ -129,6 +130,7 @@ extern int vfp_restore_user_hwstate(struct user_vfp *, |
| #define TIF_NOTIFY_RESUME 2 /* callback before returning to user */ |
| #define TIF_UPROBE 3 /* breakpointed or singlestepping */ |
| #define TIF_NOTIFY_SIGNAL 4 /* signal notifications exist */ |
| +#define TIF_NEED_RESCHED_LAZY 5 |
| |
| #define TIF_USING_IWMMXT 17 |
| #define TIF_MEMDIE 18 /* is terminating due to OOM killer */ |
| @@ -148,6 +150,7 @@ extern int vfp_restore_user_hwstate(struct user_vfp *, |
| #define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT) |
| #define _TIF_SECCOMP (1 << TIF_SECCOMP) |
| #define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) |
| +#define _TIF_NEED_RESCHED_LAZY (1 << TIF_NEED_RESCHED_LAZY) |
| #define _TIF_USING_IWMMXT (1 << TIF_USING_IWMMXT) |
| |
| /* Checks for any syscall work in entry-common.S */ |
| @@ -157,7 +160,8 @@ extern int vfp_restore_user_hwstate(struct user_vfp *, |
| /* |
| * Change these and you break ASM code in entry-common.S |
| */ |
| -#define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \ |
| +#define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY | \ |
| + _TIF_SIGPENDING | \ |
| _TIF_NOTIFY_RESUME | _TIF_UPROBE | \ |
| _TIF_NOTIFY_SIGNAL) |
| |
| diff --git a/arch/arm/kernel/asm-offsets.c b/arch/arm/kernel/asm-offsets.c |
| index 2c8d76fd7c66..c3bdec7d2df9 100644 |
| --- a/arch/arm/kernel/asm-offsets.c |
| +++ b/arch/arm/kernel/asm-offsets.c |
| @@ -43,6 +43,7 @@ int main(void) |
| BLANK(); |
| DEFINE(TI_FLAGS, offsetof(struct thread_info, flags)); |
| DEFINE(TI_PREEMPT, offsetof(struct thread_info, preempt_count)); |
| + DEFINE(TI_PREEMPT_LAZY, offsetof(struct thread_info, preempt_lazy_count)); |
| DEFINE(TI_CPU, offsetof(struct thread_info, cpu)); |
| DEFINE(TI_CPU_DOMAIN, offsetof(struct thread_info, cpu_domain)); |
| DEFINE(TI_CPU_SAVE, offsetof(struct thread_info, cpu_context)); |
| diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S |
| index c39303e5c234..cfb4660e9fea 100644 |
| --- a/arch/arm/kernel/entry-armv.S |
| +++ b/arch/arm/kernel/entry-armv.S |
| @@ -222,11 +222,18 @@ ENDPROC(__dabt_svc) |
| |
| #ifdef CONFIG_PREEMPTION |
| ldr r8, [tsk, #TI_PREEMPT] @ get preempt count |
| - ldr r0, [tsk, #TI_FLAGS] @ get flags |
| teq r8, #0 @ if preempt count != 0 |
| + bne 1f @ return from exeption |
| + ldr r0, [tsk, #TI_FLAGS] @ get flags |
| + tst r0, #_TIF_NEED_RESCHED @ if NEED_RESCHED is set |
| + blne svc_preempt @ preempt! |
| + |
| + ldr r8, [tsk, #TI_PREEMPT_LAZY] @ get preempt lazy count |
| + teq r8, #0 @ if preempt lazy count != 0 |
| movne r0, #0 @ force flags to 0 |
| - tst r0, #_TIF_NEED_RESCHED |
| + tst r0, #_TIF_NEED_RESCHED_LAZY |
| blne svc_preempt |
| +1: |
| #endif |
| |
| svc_exit r5, irq = 1 @ return from exception |
| @@ -241,8 +248,14 @@ ENDPROC(__irq_svc) |
| 1: bl preempt_schedule_irq @ irq en/disable is done inside |
| ldr r0, [tsk, #TI_FLAGS] @ get new tasks TI_FLAGS |
| tst r0, #_TIF_NEED_RESCHED |
| + bne 1b |
| + tst r0, #_TIF_NEED_RESCHED_LAZY |
| reteq r8 @ go again |
| - b 1b |
| + ldr r0, [tsk, #TI_PREEMPT_LAZY] @ get preempt lazy count |
| + teq r0, #0 @ if preempt lazy count != 0 |
| + beq 1b |
| + ret r8 @ go again |
| + |
| #endif |
| |
| __und_fault: |
| diff --git a/arch/arm/kernel/signal.c b/arch/arm/kernel/signal.c |
| index e07f359254c3..b50a3248e79f 100644 |
| --- a/arch/arm/kernel/signal.c |
| +++ b/arch/arm/kernel/signal.c |
| @@ -607,7 +607,8 @@ do_work_pending(struct pt_regs *regs, unsigned int thread_flags, int syscall) |
| */ |
| trace_hardirqs_off(); |
| do { |
| - if (likely(thread_flags & _TIF_NEED_RESCHED)) { |
| + if (likely(thread_flags & (_TIF_NEED_RESCHED | |
| + _TIF_NEED_RESCHED_LAZY))) { |
| schedule(); |
| } else { |
| if (unlikely(!user_mode(regs))) |
| diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c |
| index b0db85310331..77877dcb54ed 100644 |
| --- a/arch/arm/mm/fault.c |
| +++ b/arch/arm/mm/fault.c |
| @@ -400,6 +400,9 @@ do_translation_fault(unsigned long addr, unsigned int fsr, |
| if (addr < TASK_SIZE) |
| return do_page_fault(addr, fsr, regs); |
| |
| + if (interrupts_enabled(regs)) |
| + local_irq_enable(); |
| + |
| if (user_mode(regs)) |
| goto bad_area; |
| |
| @@ -470,6 +473,9 @@ do_translation_fault(unsigned long addr, unsigned int fsr, |
| static int |
| do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) |
| { |
| + if (interrupts_enabled(regs)) |
| + local_irq_enable(); |
| + |
| do_bad_area(addr, fsr, regs); |
| return 0; |
| } |
| diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig |
| index c15f71501c6c..0470165e2d30 100644 |
| --- a/arch/arm64/Kconfig |
| +++ b/arch/arm64/Kconfig |
| @@ -93,6 +93,7 @@ config ARM64 |
| select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 |
| select ARCH_SUPPORTS_NUMA_BALANCING |
| select ARCH_SUPPORTS_PAGE_TABLE_CHECK |
| + select ARCH_SUPPORTS_RT |
| select ARCH_WANT_COMPAT_IPC_PARSE_VERSION if COMPAT |
| select ARCH_WANT_DEFAULT_BPF_JIT |
| select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT |
| @@ -199,6 +200,7 @@ config ARM64 |
| select HAVE_PERF_USER_STACK_DUMP |
| select HAVE_PREEMPT_DYNAMIC_KEY |
| select HAVE_REGS_AND_STACK_ACCESS_API |
| + select HAVE_PREEMPT_LAZY |
| select HAVE_POSIX_CPU_TIMERS_TASK_WORK |
| select HAVE_FUNCTION_ARG_ACCESS_API |
| select MMU_GATHER_RCU_TABLE_FREE |
| diff --git a/arch/arm64/include/asm/preempt.h b/arch/arm64/include/asm/preempt.h |
| index 0159b625cc7f..a5486918e5ee 100644 |
| --- a/arch/arm64/include/asm/preempt.h |
| +++ b/arch/arm64/include/asm/preempt.h |
| @@ -71,13 +71,36 @@ static inline bool __preempt_count_dec_and_test(void) |
| * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE |
| * pair. |
| */ |
| - return !pc || !READ_ONCE(ti->preempt_count); |
| + if (!pc || !READ_ONCE(ti->preempt_count)) |
| + return true; |
| +#ifdef CONFIG_PREEMPT_LAZY |
| + if ((pc & ~PREEMPT_NEED_RESCHED)) |
| + return false; |
| + if (current_thread_info()->preempt_lazy_count) |
| + return false; |
| + return test_thread_flag(TIF_NEED_RESCHED_LAZY); |
| +#else |
| + return false; |
| +#endif |
| } |
| |
| static inline bool should_resched(int preempt_offset) |
| { |
| +#ifdef CONFIG_PREEMPT_LAZY |
| + u64 pc = READ_ONCE(current_thread_info()->preempt_count); |
| + if (pc == preempt_offset) |
| + return true; |
| + |
| + if ((pc & ~PREEMPT_NEED_RESCHED) != preempt_offset) |
| + return false; |
| + |
| + if (current_thread_info()->preempt_lazy_count) |
| + return false; |
| + return test_thread_flag(TIF_NEED_RESCHED_LAZY); |
| +#else |
| u64 pc = READ_ONCE(current_thread_info()->preempt_count); |
| return pc == preempt_offset; |
| +#endif |
| } |
| |
| #ifdef CONFIG_PREEMPTION |
| diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h |
| index 848739c15de8..4b7148fd5551 100644 |
| --- a/arch/arm64/include/asm/thread_info.h |
| +++ b/arch/arm64/include/asm/thread_info.h |
| @@ -26,6 +26,7 @@ struct thread_info { |
| #ifdef CONFIG_ARM64_SW_TTBR0_PAN |
| u64 ttbr0; /* saved TTBR0_EL1 */ |
| #endif |
| + int preempt_lazy_count; /* 0 => preemptable, <0 => bug */ |
| union { |
| u64 preempt_count; /* 0 => preemptible, <0 => bug */ |
| struct { |
| @@ -68,6 +69,7 @@ int arch_dup_task_struct(struct task_struct *dst, |
| #define TIF_UPROBE 4 /* uprobe breakpoint or singlestep */ |
| #define TIF_MTE_ASYNC_FAULT 5 /* MTE Asynchronous Tag Check Fault */ |
| #define TIF_NOTIFY_SIGNAL 6 /* signal notifications exist */ |
| +#define TIF_NEED_RESCHED_LAZY 7 |
| #define TIF_SYSCALL_TRACE 8 /* syscall trace active */ |
| #define TIF_SYSCALL_AUDIT 9 /* syscall auditing */ |
| #define TIF_SYSCALL_TRACEPOINT 10 /* syscall tracepoint for ftrace */ |
| @@ -100,8 +102,10 @@ int arch_dup_task_struct(struct task_struct *dst, |
| #define _TIF_SVE (1 << TIF_SVE) |
| #define _TIF_MTE_ASYNC_FAULT (1 << TIF_MTE_ASYNC_FAULT) |
| #define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) |
| +#define _TIF_NEED_RESCHED_LAZY (1 << TIF_NEED_RESCHED_LAZY) |
| |
| -#define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \ |
| +#define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY | \ |
| + _TIF_SIGPENDING | \ |
| _TIF_NOTIFY_RESUME | _TIF_FOREIGN_FPSTATE | \ |
| _TIF_UPROBE | _TIF_MTE_ASYNC_FAULT | \ |
| _TIF_NOTIFY_SIGNAL) |
| @@ -110,6 +114,8 @@ int arch_dup_task_struct(struct task_struct *dst, |
| _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP | \ |
| _TIF_SYSCALL_EMU) |
| |
| +#define _TIF_NEED_RESCHED_MASK (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY) |
| + |
| #ifdef CONFIG_SHADOW_CALL_STACK |
| #define INIT_SCS \ |
| .scs_base = init_shadow_call_stack, \ |
| diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c |
| index 1197e7679882..e74c0415f67e 100644 |
| --- a/arch/arm64/kernel/asm-offsets.c |
| +++ b/arch/arm64/kernel/asm-offsets.c |
| @@ -32,6 +32,7 @@ int main(void) |
| DEFINE(TSK_TI_CPU, offsetof(struct task_struct, thread_info.cpu)); |
| DEFINE(TSK_TI_FLAGS, offsetof(struct task_struct, thread_info.flags)); |
| DEFINE(TSK_TI_PREEMPT, offsetof(struct task_struct, thread_info.preempt_count)); |
| + DEFINE(TSK_TI_PREEMPT_LAZY, offsetof(struct task_struct, thread_info.preempt_lazy_count)); |
| #ifdef CONFIG_ARM64_SW_TTBR0_PAN |
| DEFINE(TSK_TI_TTBR0, offsetof(struct task_struct, thread_info.ttbr0)); |
| #endif |
| diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c |
| index 82f4572c8ddf..2a606c7bf025 100644 |
| --- a/arch/arm64/kernel/signal.c |
| +++ b/arch/arm64/kernel/signal.c |
| @@ -1108,7 +1108,7 @@ static void do_signal(struct pt_regs *regs) |
| void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags) |
| { |
| do { |
| - if (thread_flags & _TIF_NEED_RESCHED) { |
| + if (thread_flags & _TIF_NEED_RESCHED_MASK) { |
| /* Unmask Debug and SError for the next task */ |
| local_daif_restore(DAIF_PROCCTX_NOIRQ); |
| |
| diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig |
| index 6050e6e10d32..df697d3f68cd 100644 |
| --- a/arch/powerpc/Kconfig |
| +++ b/arch/powerpc/Kconfig |
| @@ -151,6 +151,7 @@ config PPC |
| select ARCH_STACKWALK |
| select ARCH_SUPPORTS_ATOMIC_RMW |
| select ARCH_SUPPORTS_DEBUG_PAGEALLOC if PPC_BOOK3S || PPC_8xx || 40x |
| + select ARCH_SUPPORTS_RT if HAVE_POSIX_CPU_TIMERS_TASK_WORK |
| select ARCH_USE_BUILTIN_BSWAP |
| select ARCH_USE_CMPXCHG_LOCKREF if PPC64 |
| select ARCH_USE_MEMTEST |
| @@ -242,8 +243,10 @@ config PPC |
| select HAVE_PERF_EVENTS_NMI if PPC64 |
| select HAVE_PERF_REGS |
| select HAVE_PERF_USER_STACK_DUMP |
| + select HAVE_PREEMPT_LAZY |
| select HAVE_REGS_AND_STACK_ACCESS_API |
| select HAVE_RELIABLE_STACKTRACE |
| + select HAVE_POSIX_CPU_TIMERS_TASK_WORK if !KVM |
| select HAVE_RSEQ |
| select HAVE_SETUP_PER_CPU_AREA if PPC64 |
| select HAVE_SOFTIRQ_ON_OWN_STACK |
| diff --git a/arch/powerpc/include/asm/stackprotector.h b/arch/powerpc/include/asm/stackprotector.h |
| index 1c8460e23583..b1653c160bab 100644 |
| --- a/arch/powerpc/include/asm/stackprotector.h |
| +++ b/arch/powerpc/include/asm/stackprotector.h |
| @@ -24,7 +24,11 @@ static __always_inline void boot_init_stack_canary(void) |
| unsigned long canary; |
| |
| /* Try to get a semi random initial value. */ |
| +#ifdef CONFIG_PREEMPT_RT |
| + canary = (unsigned long)&canary; |
| +#else |
| canary = get_random_canary(); |
| +#endif |
| canary ^= mftb(); |
| canary ^= LINUX_VERSION_CODE; |
| canary &= CANARY_MASK; |
| diff --git a/arch/powerpc/include/asm/thread_info.h b/arch/powerpc/include/asm/thread_info.h |
| index c4b798aa6ce8..923621486811 100644 |
| --- a/arch/powerpc/include/asm/thread_info.h |
| +++ b/arch/powerpc/include/asm/thread_info.h |
| @@ -53,6 +53,8 @@ |
| struct thread_info { |
| int preempt_count; /* 0 => preemptable, |
| <0 => BUG */ |
| + int preempt_lazy_count; /* 0 => preemptable, |
| + <0 => BUG */ |
| #ifdef CONFIG_SMP |
| unsigned int cpu; |
| #endif |
| @@ -77,6 +79,7 @@ struct thread_info { |
| #define INIT_THREAD_INFO(tsk) \ |
| { \ |
| .preempt_count = INIT_PREEMPT_COUNT, \ |
| + .preempt_lazy_count = 0, \ |
| .flags = 0, \ |
| } |
| |
| @@ -102,6 +105,7 @@ void arch_setup_new_exec(void); |
| #define TIF_PATCH_PENDING 6 /* pending live patching update */ |
| #define TIF_SYSCALL_AUDIT 7 /* syscall auditing active */ |
| #define TIF_SINGLESTEP 8 /* singlestepping active */ |
| +#define TIF_NEED_RESCHED_LAZY 9 /* lazy rescheduling necessary */ |
| #define TIF_SECCOMP 10 /* secure computing */ |
| #define TIF_RESTOREALL 11 /* Restore all regs (implies NOERROR) */ |
| #define TIF_NOERROR 12 /* Force successful syscall return */ |
| @@ -117,6 +121,7 @@ void arch_setup_new_exec(void); |
| #define TIF_POLLING_NRFLAG 19 /* true if poll_idle() is polling TIF_NEED_RESCHED */ |
| #define TIF_32BIT 20 /* 32 bit binary */ |
| |
| + |
| /* as above, but as bit values */ |
| #define _TIF_SYSCALL_TRACE (1<<TIF_SYSCALL_TRACE) |
| #define _TIF_SIGPENDING (1<<TIF_SIGPENDING) |
| @@ -128,6 +133,7 @@ void arch_setup_new_exec(void); |
| #define _TIF_PATCH_PENDING (1<<TIF_PATCH_PENDING) |
| #define _TIF_SYSCALL_AUDIT (1<<TIF_SYSCALL_AUDIT) |
| #define _TIF_SINGLESTEP (1<<TIF_SINGLESTEP) |
| +#define _TIF_NEED_RESCHED_LAZY (1<<TIF_NEED_RESCHED_LAZY) |
| #define _TIF_SECCOMP (1<<TIF_SECCOMP) |
| #define _TIF_RESTOREALL (1<<TIF_RESTOREALL) |
| #define _TIF_NOERROR (1<<TIF_NOERROR) |
| @@ -141,10 +147,12 @@ void arch_setup_new_exec(void); |
| _TIF_SYSCALL_EMU) |
| |
| #define _TIF_USER_WORK_MASK (_TIF_SIGPENDING | _TIF_NEED_RESCHED | \ |
| + _TIF_NEED_RESCHED_LAZY | \ |
| _TIF_NOTIFY_RESUME | _TIF_UPROBE | \ |
| _TIF_RESTORE_TM | _TIF_PATCH_PENDING | \ |
| _TIF_NOTIFY_SIGNAL) |
| #define _TIF_PERSYSCALL_MASK (_TIF_RESTOREALL|_TIF_NOERROR) |
| +#define _TIF_NEED_RESCHED_MASK (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY) |
| |
| /* Bits in local_flags */ |
| /* Don't move TLF_NAPPING without adjusting the code in entry_32.S */ |
| diff --git a/arch/powerpc/kernel/interrupt.c b/arch/powerpc/kernel/interrupt.c |
| index cf770d86c03c..2c454731c250 100644 |
| --- a/arch/powerpc/kernel/interrupt.c |
| +++ b/arch/powerpc/kernel/interrupt.c |
| @@ -186,7 +186,7 @@ interrupt_exit_user_prepare_main(unsigned long ret, struct pt_regs *regs) |
| ti_flags = read_thread_flags(); |
| while (unlikely(ti_flags & (_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM))) { |
| local_irq_enable(); |
| - if (ti_flags & _TIF_NEED_RESCHED) { |
| + if (ti_flags & _TIF_NEED_RESCHED_MASK) { |
| schedule(); |
| } else { |
| /* |
| @@ -397,11 +397,15 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs) |
| /* Returning to a kernel context with local irqs enabled. */ |
| WARN_ON_ONCE(!(regs->msr & MSR_EE)); |
| again: |
| - if (IS_ENABLED(CONFIG_PREEMPT)) { |
| + if (IS_ENABLED(CONFIG_PREEMPTION)) { |
| /* Return to preemptible kernel context */ |
| if (unlikely(read_thread_flags() & _TIF_NEED_RESCHED)) { |
| if (preempt_count() == 0) |
| preempt_schedule_irq(); |
| + } else if (unlikely(current_thread_info()->flags & _TIF_NEED_RESCHED_LAZY)) { |
| + if ((preempt_count() == 0) && |
| + (current_thread_info()->preempt_lazy_count == 0)) |
| + preempt_schedule_irq(); |
| } |
| } |
| |
| diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c |
| index 362b712386f6..ddf5b716394f 100644 |
| --- a/arch/powerpc/kernel/traps.c |
| +++ b/arch/powerpc/kernel/traps.c |
| @@ -261,12 +261,17 @@ static char *get_mmu_str(void) |
| |
| static int __die(const char *str, struct pt_regs *regs, long err) |
| { |
| + const char *pr = ""; |
| + |
| printk("Oops: %s, sig: %ld [#%d]\n", str, err, ++die_counter); |
| |
| + if (IS_ENABLED(CONFIG_PREEMPTION)) |
| + pr = IS_ENABLED(CONFIG_PREEMPT_RT) ? " PREEMPT_RT" : " PREEMPT"; |
| + |
| printk("%s PAGE_SIZE=%luK%s%s%s%s%s%s %s\n", |
| IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN) ? "LE" : "BE", |
| PAGE_SIZE / 1024, get_mmu_str(), |
| - IS_ENABLED(CONFIG_PREEMPT) ? " PREEMPT" : "", |
| + pr, |
| IS_ENABLED(CONFIG_SMP) ? " SMP" : "", |
| IS_ENABLED(CONFIG_SMP) ? (" NR_CPUS=" __stringify(NR_CPUS)) : "", |
| debug_pagealloc_enabled() ? " DEBUG_PAGEALLOC" : "", |
| diff --git a/arch/powerpc/kvm/Kconfig b/arch/powerpc/kvm/Kconfig |
| index a9f57dad6d91..a0b528d4bb7c 100644 |
| --- a/arch/powerpc/kvm/Kconfig |
| +++ b/arch/powerpc/kvm/Kconfig |
| @@ -225,6 +225,7 @@ config KVM_E500MC |
| config KVM_MPIC |
| bool "KVM in-kernel MPIC emulation" |
| depends on KVM && PPC_E500 |
| + depends on !PREEMPT_RT |
| select HAVE_KVM_IRQCHIP |
| select HAVE_KVM_IRQFD |
| select HAVE_KVM_IRQ_ROUTING |
| diff --git a/arch/powerpc/platforms/pseries/iommu.c b/arch/powerpc/platforms/pseries/iommu.c |
| index 1e5f083cdb72..b55d2bc7e40f 100644 |
| --- a/arch/powerpc/platforms/pseries/iommu.c |
| +++ b/arch/powerpc/platforms/pseries/iommu.c |
| @@ -24,6 +24,7 @@ |
| #include <linux/of.h> |
| #include <linux/iommu.h> |
| #include <linux/rculist.h> |
| +#include <linux/local_lock.h> |
| #include <asm/io.h> |
| #include <asm/prom.h> |
| #include <asm/rtas.h> |
| @@ -200,7 +201,13 @@ static int tce_build_pSeriesLP(unsigned long liobn, long tcenum, long tceshift, |
| return ret; |
| } |
| |
| -static DEFINE_PER_CPU(__be64 *, tce_page); |
| +struct tce_page { |
| + __be64 * page; |
| + local_lock_t lock; |
| +}; |
| +static DEFINE_PER_CPU(struct tce_page, tce_page) = { |
| + .lock = INIT_LOCAL_LOCK(lock), |
| +}; |
| |
| static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum, |
| long npages, unsigned long uaddr, |
| @@ -223,9 +230,10 @@ static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum, |
| direction, attrs); |
| } |
| |
| - local_irq_save(flags); /* to protect tcep and the page behind it */ |
| + /* to protect tcep and the page behind it */ |
| + local_lock_irqsave(&tce_page.lock, flags); |
| |
| - tcep = __this_cpu_read(tce_page); |
| + tcep = __this_cpu_read(tce_page.page); |
| |
| /* This is safe to do since interrupts are off when we're called |
| * from iommu_alloc{,_sg}() |
| @@ -234,12 +242,12 @@ static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum, |
| tcep = (__be64 *)__get_free_page(GFP_ATOMIC); |
| /* If allocation fails, fall back to the loop implementation */ |
| if (!tcep) { |
| - local_irq_restore(flags); |
| + local_unlock_irqrestore(&tce_page.lock, flags); |
| return tce_build_pSeriesLP(tbl->it_index, tcenum, |
| tceshift, |
| npages, uaddr, direction, attrs); |
| } |
| - __this_cpu_write(tce_page, tcep); |
| + __this_cpu_write(tce_page.page, tcep); |
| } |
| |
| rpn = __pa(uaddr) >> tceshift; |
| @@ -269,7 +277,7 @@ static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum, |
| tcenum += limit; |
| } while (npages > 0 && !rc); |
| |
| - local_irq_restore(flags); |
| + local_unlock_irqrestore(&tce_page.lock, flags); |
| |
| if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) { |
| ret = (int)rc; |
| @@ -454,16 +462,17 @@ static int tce_setrange_multi_pSeriesLP(unsigned long start_pfn, |
| DMA_BIDIRECTIONAL, 0); |
| } |
| |
| - local_irq_disable(); /* to protect tcep and the page behind it */ |
| - tcep = __this_cpu_read(tce_page); |
| + /* to protect tcep and the page behind it */ |
| + local_lock_irq(&tce_page.lock); |
| + tcep = __this_cpu_read(tce_page.page); |
| |
| if (!tcep) { |
| tcep = (__be64 *)__get_free_page(GFP_ATOMIC); |
| if (!tcep) { |
| - local_irq_enable(); |
| + local_unlock_irq(&tce_page.lock); |
| return -ENOMEM; |
| } |
| - __this_cpu_write(tce_page, tcep); |
| + __this_cpu_write(tce_page.page, tcep); |
| } |
| |
| proto_tce = TCE_PCI_READ | TCE_PCI_WRITE; |
| @@ -506,7 +515,7 @@ static int tce_setrange_multi_pSeriesLP(unsigned long start_pfn, |
| |
| /* error cleanup: caller will clear whole range */ |
| |
| - local_irq_enable(); |
| + local_unlock_irq(&tce_page.lock); |
| return rc; |
| } |
| |
| diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig |
| index 5caa023e9839..cf6106b60b4a 100644 |
| --- a/arch/x86/Kconfig |
| +++ b/arch/x86/Kconfig |
| @@ -113,6 +113,7 @@ config X86 |
| select ARCH_USES_CFI_TRAPS if X86_64 && CFI_CLANG |
| select ARCH_SUPPORTS_LTO_CLANG |
| select ARCH_SUPPORTS_LTO_CLANG_THIN |
| + select ARCH_SUPPORTS_RT |
| select ARCH_USE_BUILTIN_BSWAP |
| select ARCH_USE_MEMTEST |
| select ARCH_USE_QUEUED_RWLOCKS |
| @@ -250,6 +251,7 @@ config X86 |
| select HAVE_PCI |
| select HAVE_PERF_REGS |
| select HAVE_PERF_USER_STACK_DUMP |
| + select HAVE_PREEMPT_LAZY |
| select MMU_GATHER_RCU_TABLE_FREE if PARAVIRT |
| select MMU_GATHER_MERGE_VMAS |
| select HAVE_POSIX_CPU_TIMERS_TASK_WORK |
| diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h |
| index 5f6daea1ee24..cd20b4a5719a 100644 |
| --- a/arch/x86/include/asm/preempt.h |
| +++ b/arch/x86/include/asm/preempt.h |
| @@ -90,17 +90,48 @@ static __always_inline void __preempt_count_sub(int val) |
| * a decrement which hits zero means we have no preempt_count and should |
| * reschedule. |
| */ |
| -static __always_inline bool __preempt_count_dec_and_test(void) |
| +static __always_inline bool ____preempt_count_dec_and_test(void) |
| { |
| return GEN_UNARY_RMWcc("decl", __preempt_count, e, __percpu_arg([var])); |
| } |
| |
| +static __always_inline bool __preempt_count_dec_and_test(void) |
| +{ |
| + if (____preempt_count_dec_and_test()) |
| + return true; |
| +#ifdef CONFIG_PREEMPT_LAZY |
| + if (preempt_count()) |
| + return false; |
| + if (current_thread_info()->preempt_lazy_count) |
| + return false; |
| + return test_thread_flag(TIF_NEED_RESCHED_LAZY); |
| +#else |
| + return false; |
| +#endif |
| +} |
| + |
| /* |
| * Returns true when we need to resched and can (barring IRQ state). |
| */ |
| static __always_inline bool should_resched(int preempt_offset) |
| { |
| +#ifdef CONFIG_PREEMPT_LAZY |
| + u32 tmp; |
| + tmp = raw_cpu_read_4(__preempt_count); |
| + if (tmp == preempt_offset) |
| + return true; |
| + |
| + /* preempt count == 0 ? */ |
| + tmp &= ~PREEMPT_NEED_RESCHED; |
| + if (tmp != preempt_offset) |
| + return false; |
| + /* XXX PREEMPT_LOCK_OFFSET */ |
| + if (current_thread_info()->preempt_lazy_count) |
| + return false; |
| + return test_thread_flag(TIF_NEED_RESCHED_LAZY); |
| +#else |
| return unlikely(raw_cpu_read_4(__preempt_count) == preempt_offset); |
| +#endif |
| } |
| |
| #ifdef CONFIG_PREEMPTION |
| diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h |
| index f0cb881c1d69..fd8fb76f324f 100644 |
| --- a/arch/x86/include/asm/thread_info.h |
| +++ b/arch/x86/include/asm/thread_info.h |
| @@ -57,6 +57,8 @@ struct thread_info { |
| unsigned long flags; /* low level flags */ |
| unsigned long syscall_work; /* SYSCALL_WORK_ flags */ |
| u32 status; /* thread synchronous flags */ |
| + int preempt_lazy_count; /* 0 => lazy preemptable |
| + <0 => BUG */ |
| #ifdef CONFIG_SMP |
| u32 cpu; /* current CPU */ |
| #endif |
| @@ -65,6 +67,7 @@ struct thread_info { |
| #define INIT_THREAD_INFO(tsk) \ |
| { \ |
| .flags = 0, \ |
| + .preempt_lazy_count = 0, \ |
| } |
| |
| #else /* !__ASSEMBLY__ */ |
| @@ -92,6 +95,7 @@ struct thread_info { |
| #define TIF_NOCPUID 15 /* CPUID is not accessible in userland */ |
| #define TIF_NOTSC 16 /* TSC is not accessible in userland */ |
| #define TIF_NOTIFY_SIGNAL 17 /* signal notifications exist */ |
| +#define TIF_NEED_RESCHED_LAZY 19 /* lazy rescheduling necessary */ |
| #define TIF_MEMDIE 20 /* is terminating due to OOM killer */ |
| #define TIF_POLLING_NRFLAG 21 /* idle is polling for TIF_NEED_RESCHED */ |
| #define TIF_IO_BITMAP 22 /* uses I/O bitmap */ |
| @@ -115,6 +119,7 @@ struct thread_info { |
| #define _TIF_NOCPUID (1 << TIF_NOCPUID) |
| #define _TIF_NOTSC (1 << TIF_NOTSC) |
| #define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) |
| +#define _TIF_NEED_RESCHED_LAZY (1 << TIF_NEED_RESCHED_LAZY) |
| #define _TIF_POLLING_NRFLAG (1 << TIF_POLLING_NRFLAG) |
| #define _TIF_IO_BITMAP (1 << TIF_IO_BITMAP) |
| #define _TIF_SPEC_FORCE_UPDATE (1 << TIF_SPEC_FORCE_UPDATE) |
| diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c |
| index 966aab902d19..ee69e4443691 100644 |
| --- a/drivers/block/zram/zram_drv.c |
| +++ b/drivers/block/zram/zram_drv.c |
| @@ -57,6 +57,40 @@ static void zram_free_page(struct zram *zram, size_t index); |
| static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec, |
| u32 index, int offset, struct bio *bio); |
| |
| +#ifdef CONFIG_PREEMPT_RT |
| +static void zram_meta_init_table_locks(struct zram *zram, size_t num_pages) |
| +{ |
| + size_t index; |
| + |
| + for (index = 0; index < num_pages; index++) |
| + spin_lock_init(&zram->table[index].lock); |
| +} |
| + |
| +static int zram_slot_trylock(struct zram *zram, u32 index) |
| +{ |
| + int ret; |
| + |
| + ret = spin_trylock(&zram->table[index].lock); |
| + if (ret) |
| + __set_bit(ZRAM_LOCK, &zram->table[index].flags); |
| + return ret; |
| +} |
| + |
| +static void zram_slot_lock(struct zram *zram, u32 index) |
| +{ |
| + spin_lock(&zram->table[index].lock); |
| + __set_bit(ZRAM_LOCK, &zram->table[index].flags); |
| +} |
| + |
| +static void zram_slot_unlock(struct zram *zram, u32 index) |
| +{ |
| + __clear_bit(ZRAM_LOCK, &zram->table[index].flags); |
| + spin_unlock(&zram->table[index].lock); |
| +} |
| + |
| +#else |
| + |
| +static void zram_meta_init_table_locks(struct zram *zram, size_t num_pages) { } |
| |
| static int zram_slot_trylock(struct zram *zram, u32 index) |
| { |
| @@ -72,6 +106,7 @@ static void zram_slot_unlock(struct zram *zram, u32 index) |
| { |
| bit_spin_unlock(ZRAM_LOCK, &zram->table[index].flags); |
| } |
| +#endif |
| |
| static inline bool init_done(struct zram *zram) |
| { |
| @@ -1187,6 +1222,7 @@ static bool zram_meta_alloc(struct zram *zram, u64 disksize) |
| |
| if (!huge_class_size) |
| huge_class_size = zs_huge_class_size(zram->mem_pool); |
| + zram_meta_init_table_locks(zram, num_pages); |
| return true; |
| } |
| |
| diff --git a/drivers/block/zram/zram_drv.h b/drivers/block/zram/zram_drv.h |
| index a2bda53020fd..ae7950b26db5 100644 |
| --- a/drivers/block/zram/zram_drv.h |
| +++ b/drivers/block/zram/zram_drv.h |
| @@ -62,6 +62,9 @@ struct zram_table_entry { |
| unsigned long element; |
| }; |
| unsigned long flags; |
| +#ifdef CONFIG_PREEMPT_RT |
| + spinlock_t lock; |
| +#endif |
| #ifdef CONFIG_ZRAM_MEMORY_TRACKING |
| ktime_t ac_time; |
| #endif |
| diff --git a/drivers/char/tpm/tpm_tis.c b/drivers/char/tpm/tpm_tis.c |
| index 0d084d6652c4..5d620322bdc2 100644 |
| --- a/drivers/char/tpm/tpm_tis.c |
| +++ b/drivers/char/tpm/tpm_tis.c |
| @@ -50,6 +50,31 @@ static inline struct tpm_tis_tcg_phy *to_tpm_tis_tcg_phy(struct tpm_tis_data *da |
| return container_of(data, struct tpm_tis_tcg_phy, priv); |
| } |
| |
| +#ifdef CONFIG_PREEMPT_RT |
| +/* |
| + * Flushes previous write operations to chip so that a subsequent |
| + * ioread*()s won't stall a cpu. |
| + */ |
| +static inline void tpm_tis_flush(void __iomem *iobase) |
| +{ |
| + ioread8(iobase + TPM_ACCESS(0)); |
| +} |
| +#else |
| +#define tpm_tis_flush(iobase) do { } while (0) |
| +#endif |
| + |
| +static inline void tpm_tis_iowrite8(u8 b, void __iomem *iobase, u32 addr) |
| +{ |
| + iowrite8(b, iobase + addr); |
| + tpm_tis_flush(iobase); |
| +} |
| + |
| +static inline void tpm_tis_iowrite32(u32 b, void __iomem *iobase, u32 addr) |
| +{ |
| + iowrite32(b, iobase + addr); |
| + tpm_tis_flush(iobase); |
| +} |
| + |
| static int interrupts = -1; |
| module_param(interrupts, int, 0444); |
| MODULE_PARM_DESC(interrupts, "Enable interrupts"); |
| @@ -202,12 +227,12 @@ static int tpm_tcg_write_bytes(struct tpm_tis_data *data, u32 addr, u16 len, |
| switch (io_mode) { |
| case TPM_TIS_PHYS_8: |
| while (len--) |
| - iowrite8(*value++, phy->iobase + addr); |
| + tpm_tis_iowrite8(*value++, phy->iobase, addr); |
| break; |
| case TPM_TIS_PHYS_16: |
| return -EINVAL; |
| case TPM_TIS_PHYS_32: |
| - iowrite32(le32_to_cpu(*((__le32 *)value)), phy->iobase + addr); |
| + tpm_tis_iowrite32(le32_to_cpu(*((__le32 *)value)), phy->iobase, addr); |
| break; |
| } |
| |
| diff --git a/drivers/gpu/drm/i915/Kconfig b/drivers/gpu/drm/i915/Kconfig |
| index 6b10868ec72f..1fbdb7b4e6e1 100644 |
| --- a/drivers/gpu/drm/i915/Kconfig |
| +++ b/drivers/gpu/drm/i915/Kconfig |
| @@ -3,7 +3,6 @@ config DRM_I915 |
| tristate "Intel 8xx/9xx/G3x/G4x/HD Graphics" |
| depends on DRM |
| depends on X86 && PCI |
| - depends on !PREEMPT_RT |
| select INTEL_GTT if X86 |
| select INTERVAL_TREE |
| # we need shmfs for the swappable backing store, and in particular |
| diff --git a/drivers/gpu/drm/i915/display/intel_crtc.c b/drivers/gpu/drm/i915/display/intel_crtc.c |
| index 6792a9056f46..43cedfef104f 100644 |
| --- a/drivers/gpu/drm/i915/display/intel_crtc.c |
| +++ b/drivers/gpu/drm/i915/display/intel_crtc.c |
| @@ -521,7 +521,8 @@ void intel_pipe_update_start(struct intel_crtc_state *new_crtc_state) |
| */ |
| intel_psr_wait_for_idle_locked(new_crtc_state); |
| |
| - local_irq_disable(); |
| + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) |
| + local_irq_disable(); |
| |
| crtc->debug.min_vbl = min; |
| crtc->debug.max_vbl = max; |
| @@ -546,11 +547,13 @@ void intel_pipe_update_start(struct intel_crtc_state *new_crtc_state) |
| break; |
| } |
| |
| - local_irq_enable(); |
| + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) |
| + local_irq_enable(); |
| |
| timeout = schedule_timeout(timeout); |
| |
| - local_irq_disable(); |
| + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) |
| + local_irq_disable(); |
| } |
| |
| finish_wait(wq, &wait); |
| @@ -583,7 +586,8 @@ void intel_pipe_update_start(struct intel_crtc_state *new_crtc_state) |
| return; |
| |
| irq_disable: |
| - local_irq_disable(); |
| + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) |
| + local_irq_disable(); |
| } |
| |
| #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_VBLANK_EVADE) |
| @@ -684,7 +688,8 @@ void intel_pipe_update_end(struct intel_crtc_state *new_crtc_state) |
| */ |
| intel_vrr_send_push(new_crtc_state); |
| |
| - local_irq_enable(); |
| + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) |
| + local_irq_enable(); |
| |
| if (intel_vgpu_active(dev_priv)) |
| return; |
| diff --git a/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c b/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c |
| index ecc990ec1b95..8d04b10681f0 100644 |
| --- a/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c |
| +++ b/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c |
| @@ -312,10 +312,9 @@ void __intel_breadcrumbs_park(struct intel_breadcrumbs *b) |
| /* Kick the work once more to drain the signalers, and disarm the irq */ |
| irq_work_sync(&b->irq_work); |
| while (READ_ONCE(b->irq_armed) && !atomic_read(&b->active)) { |
| - local_irq_disable(); |
| - signal_irq_work(&b->irq_work); |
| - local_irq_enable(); |
| + irq_work_queue(&b->irq_work); |
| cond_resched(); |
| + irq_work_sync(&b->irq_work); |
| } |
| } |
| |
| diff --git a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c |
| index f903ee1ce06e..f54059b63ea9 100644 |
| --- a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c |
| +++ b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c |
| @@ -1302,7 +1302,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) |
| * and context switches) submission. |
| */ |
| |
| - spin_lock(&sched_engine->lock); |
| + spin_lock_irq(&sched_engine->lock); |
| |
| /* |
| * If the queue is higher priority than the last |
| @@ -1402,7 +1402,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) |
| * Even if ELSP[1] is occupied and not worthy |
| * of timeslices, our queue might be. |
| */ |
| - spin_unlock(&sched_engine->lock); |
| + spin_unlock_irq(&sched_engine->lock); |
| return; |
| } |
| } |
| @@ -1428,7 +1428,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) |
| |
| if (last && !can_merge_rq(last, rq)) { |
| spin_unlock(&ve->base.sched_engine->lock); |
| - spin_unlock(&engine->sched_engine->lock); |
| + spin_unlock_irq(&engine->sched_engine->lock); |
| return; /* leave this for another sibling */ |
| } |
| |
| @@ -1590,7 +1590,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) |
| */ |
| sched_engine->queue_priority_hint = queue_prio(sched_engine); |
| i915_sched_engine_reset_on_empty(sched_engine); |
| - spin_unlock(&sched_engine->lock); |
| + spin_unlock_irq(&sched_engine->lock); |
| |
| /* |
| * We can skip poking the HW if we ended up with exactly the same set |
| @@ -1616,13 +1616,6 @@ static void execlists_dequeue(struct intel_engine_cs *engine) |
| } |
| } |
| |
| -static void execlists_dequeue_irq(struct intel_engine_cs *engine) |
| -{ |
| - local_irq_disable(); /* Suspend interrupts across request submission */ |
| - execlists_dequeue(engine); |
| - local_irq_enable(); /* flush irq_work (e.g. breadcrumb enabling) */ |
| -} |
| - |
| static void clear_ports(struct i915_request **ports, int count) |
| { |
| memset_p((void **)ports, NULL, count); |
| @@ -2476,7 +2469,7 @@ static void execlists_submission_tasklet(struct tasklet_struct *t) |
| } |
| |
| if (!engine->execlists.pending[0]) { |
| - execlists_dequeue_irq(engine); |
| + execlists_dequeue(engine); |
| start_timeslice(engine); |
| } |
| |
| diff --git a/drivers/gpu/drm/i915/gt/intel_reset.c b/drivers/gpu/drm/i915/gt/intel_reset.c |
| index 10b930eaa8cb..6108a449cd19 100644 |
| --- a/drivers/gpu/drm/i915/gt/intel_reset.c |
| +++ b/drivers/gpu/drm/i915/gt/intel_reset.c |
| @@ -174,13 +174,13 @@ static int i915_do_reset(struct intel_gt *gt, |
| /* Assert reset for at least 20 usec, and wait for acknowledgement. */ |
| pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE); |
| udelay(50); |
| - err = wait_for_atomic(i915_in_reset(pdev), 50); |
| + err = _wait_for_atomic(i915_in_reset(pdev), 50, 0); |
| |
| /* Clear the reset request. */ |
| pci_write_config_byte(pdev, I915_GDRST, 0); |
| udelay(50); |
| if (!err) |
| - err = wait_for_atomic(!i915_in_reset(pdev), 50); |
| + err = _wait_for_atomic(!i915_in_reset(pdev), 50, 0); |
| |
| return err; |
| } |
| @@ -200,7 +200,7 @@ static int g33_do_reset(struct intel_gt *gt, |
| struct pci_dev *pdev = to_pci_dev(gt->i915->drm.dev); |
| |
| pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE); |
| - return wait_for_atomic(g4x_reset_complete(pdev), 50); |
| + return _wait_for_atomic(g4x_reset_complete(pdev), 50, 0); |
| } |
| |
| static int g4x_do_reset(struct intel_gt *gt, |
| @@ -217,7 +217,7 @@ static int g4x_do_reset(struct intel_gt *gt, |
| |
| pci_write_config_byte(pdev, I915_GDRST, |
| GRDOM_MEDIA | GRDOM_RESET_ENABLE); |
| - ret = wait_for_atomic(g4x_reset_complete(pdev), 50); |
| + ret = _wait_for_atomic(g4x_reset_complete(pdev), 50, 0); |
| if (ret) { |
| GT_TRACE(gt, "Wait for media reset failed\n"); |
| goto out; |
| @@ -225,7 +225,7 @@ static int g4x_do_reset(struct intel_gt *gt, |
| |
| pci_write_config_byte(pdev, I915_GDRST, |
| GRDOM_RENDER | GRDOM_RESET_ENABLE); |
| - ret = wait_for_atomic(g4x_reset_complete(pdev), 50); |
| + ret = _wait_for_atomic(g4x_reset_complete(pdev), 50, 0); |
| if (ret) { |
| GT_TRACE(gt, "Wait for render reset failed\n"); |
| goto out; |
| @@ -718,9 +718,7 @@ int __intel_gt_reset(struct intel_gt *gt, intel_engine_mask_t engine_mask) |
| intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL); |
| for (retry = 0; ret == -ETIMEDOUT && retry < retries; retry++) { |
| GT_TRACE(gt, "engine_mask=%x\n", engine_mask); |
| - preempt_disable(); |
| ret = reset(gt, engine_mask, retry); |
| - preempt_enable(); |
| } |
| intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL); |
| |
| diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c |
| index f93ffa6626a5..6e9d033cf808 100644 |
| --- a/drivers/gpu/drm/i915/i915_irq.c |
| +++ b/drivers/gpu/drm/i915/i915_irq.c |
| @@ -917,7 +917,8 @@ static bool i915_get_crtc_scanoutpos(struct drm_crtc *_crtc, |
| */ |
| spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); |
| |
| - /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */ |
| + if (IS_ENABLED(CONFIG_PREEMPT_RT)) |
| + preempt_disable(); |
| |
| /* Get optional system timestamp before query. */ |
| if (stime) |
| @@ -981,7 +982,8 @@ static bool i915_get_crtc_scanoutpos(struct drm_crtc *_crtc, |
| if (etime) |
| *etime = ktime_get(); |
| |
| - /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */ |
| + if (IS_ENABLED(CONFIG_PREEMPT_RT)) |
| + preempt_enable(); |
| |
| spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); |
| |
| diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c |
| index 7ce126a01cbf..64a032dfaa90 100644 |
| --- a/drivers/gpu/drm/i915/i915_request.c |
| +++ b/drivers/gpu/drm/i915/i915_request.c |
| @@ -609,7 +609,6 @@ bool __i915_request_submit(struct i915_request *request) |
| |
| RQ_TRACE(request, "\n"); |
| |
| - GEM_BUG_ON(!irqs_disabled()); |
| lockdep_assert_held(&engine->sched_engine->lock); |
| |
| /* |
| @@ -718,7 +717,6 @@ void __i915_request_unsubmit(struct i915_request *request) |
| */ |
| RQ_TRACE(request, "\n"); |
| |
| - GEM_BUG_ON(!irqs_disabled()); |
| lockdep_assert_held(&engine->sched_engine->lock); |
| |
| /* |
| diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h |
| index 37b5c9e9d260..73f29d8008f0 100644 |
| --- a/drivers/gpu/drm/i915/i915_trace.h |
| +++ b/drivers/gpu/drm/i915/i915_trace.h |
| @@ -6,6 +6,10 @@ |
| #if !defined(_I915_TRACE_H_) || defined(TRACE_HEADER_MULTI_READ) |
| #define _I915_TRACE_H_ |
| |
| +#ifdef CONFIG_PREEMPT_RT |
| +#define NOTRACE |
| +#endif |
| + |
| #include <linux/stringify.h> |
| #include <linux/types.h> |
| #include <linux/tracepoint.h> |
| @@ -323,7 +327,7 @@ DEFINE_EVENT(i915_request, i915_request_add, |
| TP_ARGS(rq) |
| ); |
| |
| -#if defined(CONFIG_DRM_I915_LOW_LEVEL_TRACEPOINTS) |
| +#if defined(CONFIG_DRM_I915_LOW_LEVEL_TRACEPOINTS) && !defined(NOTRACE) |
| DEFINE_EVENT(i915_request, i915_request_guc_submit, |
| TP_PROTO(struct i915_request *rq), |
| TP_ARGS(rq) |
| diff --git a/drivers/gpu/drm/i915/i915_utils.h b/drivers/gpu/drm/i915/i915_utils.h |
| index 6c14d13364bf..de58855e6926 100644 |
| --- a/drivers/gpu/drm/i915/i915_utils.h |
| +++ b/drivers/gpu/drm/i915/i915_utils.h |
| @@ -294,7 +294,7 @@ wait_remaining_ms_from_jiffies(unsigned long timestamp_jiffies, int to_wait_ms) |
| #define wait_for(COND, MS) _wait_for((COND), (MS) * 1000, 10, 1000) |
| |
| /* If CONFIG_PREEMPT_COUNT is disabled, in_atomic() always reports false. */ |
| -#if defined(CONFIG_DRM_I915_DEBUG) && defined(CONFIG_PREEMPT_COUNT) |
| +#if defined(CONFIG_DRM_I915_DEBUG) && defined(CONFIG_PREEMPT_COUNT) && !defined(CONFIG_PREEMPT_RT) |
| # define _WAIT_FOR_ATOMIC_CHECK(ATOMIC) WARN_ON_ONCE((ATOMIC) && !in_atomic()) |
| #else |
| # define _WAIT_FOR_ATOMIC_CHECK(ATOMIC) do { } while (0) |
| diff --git a/drivers/net/ethernet/alacritech/slic.h b/drivers/net/ethernet/alacritech/slic.h |
| index 4eecbdfff3ff..82071d0e5f7f 100644 |
| --- a/drivers/net/ethernet/alacritech/slic.h |
| +++ b/drivers/net/ethernet/alacritech/slic.h |
| @@ -288,13 +288,13 @@ do { \ |
| u64_stats_update_end(&(st)->syncp); \ |
| } while (0) |
| |
| -#define SLIC_GET_STATS_COUNTER(newst, st, counter) \ |
| -{ \ |
| - unsigned int start; \ |
| +#define SLIC_GET_STATS_COUNTER(newst, st, counter) \ |
| +{ \ |
| + unsigned int start; \ |
| do { \ |
| - start = u64_stats_fetch_begin_irq(&(st)->syncp); \ |
| - newst = (st)->counter; \ |
| - } while (u64_stats_fetch_retry_irq(&(st)->syncp, start)); \ |
| + start = u64_stats_fetch_begin(&(st)->syncp); \ |
| + newst = (st)->counter; \ |
| + } while (u64_stats_fetch_retry(&(st)->syncp, start)); \ |
| } |
| |
| struct slic_upr { |
| diff --git a/drivers/net/ethernet/amazon/ena/ena_ethtool.c b/drivers/net/ethernet/amazon/ena/ena_ethtool.c |
| index 444ccef76da2..8da79eedc057 100644 |
| --- a/drivers/net/ethernet/amazon/ena/ena_ethtool.c |
| +++ b/drivers/net/ethernet/amazon/ena/ena_ethtool.c |
| @@ -118,9 +118,9 @@ static void ena_safe_update_stat(u64 *src, u64 *dst, |
| unsigned int start; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(syncp); |
| + start = u64_stats_fetch_begin(syncp); |
| *(dst) = *src; |
| - } while (u64_stats_fetch_retry_irq(syncp, start)); |
| + } while (u64_stats_fetch_retry(syncp, start)); |
| } |
| |
| static void ena_queue_stats(struct ena_adapter *adapter, u64 **data) |
| diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.c b/drivers/net/ethernet/amazon/ena/ena_netdev.c |
| index 9e82e7b9c3b7..e047cc3aeb05 100644 |
| --- a/drivers/net/ethernet/amazon/ena/ena_netdev.c |
| +++ b/drivers/net/ethernet/amazon/ena/ena_netdev.c |
| @@ -3296,10 +3296,10 @@ static void ena_get_stats64(struct net_device *netdev, |
| tx_ring = &adapter->tx_ring[i]; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&tx_ring->syncp); |
| + start = u64_stats_fetch_begin(&tx_ring->syncp); |
| packets = tx_ring->tx_stats.cnt; |
| bytes = tx_ring->tx_stats.bytes; |
| - } while (u64_stats_fetch_retry_irq(&tx_ring->syncp, start)); |
| + } while (u64_stats_fetch_retry(&tx_ring->syncp, start)); |
| |
| stats->tx_packets += packets; |
| stats->tx_bytes += bytes; |
| @@ -3307,20 +3307,20 @@ static void ena_get_stats64(struct net_device *netdev, |
| rx_ring = &adapter->rx_ring[i]; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&rx_ring->syncp); |
| + start = u64_stats_fetch_begin(&rx_ring->syncp); |
| packets = rx_ring->rx_stats.cnt; |
| bytes = rx_ring->rx_stats.bytes; |
| - } while (u64_stats_fetch_retry_irq(&rx_ring->syncp, start)); |
| + } while (u64_stats_fetch_retry(&rx_ring->syncp, start)); |
| |
| stats->rx_packets += packets; |
| stats->rx_bytes += bytes; |
| } |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&adapter->syncp); |
| + start = u64_stats_fetch_begin(&adapter->syncp); |
| rx_drops = adapter->dev_stats.rx_drops; |
| tx_drops = adapter->dev_stats.tx_drops; |
| - } while (u64_stats_fetch_retry_irq(&adapter->syncp, start)); |
| + } while (u64_stats_fetch_retry(&adapter->syncp, start)); |
| |
| stats->rx_dropped = rx_drops; |
| stats->tx_dropped = tx_drops; |
| diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_ring.c b/drivers/net/ethernet/aquantia/atlantic/aq_ring.c |
| index 0eaaba3a18ee..4f6315e5b714 100644 |
| --- a/drivers/net/ethernet/aquantia/atlantic/aq_ring.c |
| +++ b/drivers/net/ethernet/aquantia/atlantic/aq_ring.c |
| @@ -941,7 +941,7 @@ unsigned int aq_ring_fill_stats_data(struct aq_ring_s *self, u64 *data) |
| /* This data should mimic aq_ethtool_queue_rx_stat_names structure */ |
| do { |
| count = 0; |
| - start = u64_stats_fetch_begin_irq(&self->stats.rx.syncp); |
| + start = u64_stats_fetch_begin(&self->stats.rx.syncp); |
| data[count] = self->stats.rx.packets; |
| data[++count] = self->stats.rx.jumbo_packets; |
| data[++count] = self->stats.rx.lro_packets; |
| @@ -958,15 +958,15 @@ unsigned int aq_ring_fill_stats_data(struct aq_ring_s *self, u64 *data) |
| data[++count] = self->stats.rx.xdp_tx; |
| data[++count] = self->stats.rx.xdp_invalid; |
| data[++count] = self->stats.rx.xdp_redirect; |
| - } while (u64_stats_fetch_retry_irq(&self->stats.rx.syncp, start)); |
| + } while (u64_stats_fetch_retry(&self->stats.rx.syncp, start)); |
| } else { |
| /* This data should mimic aq_ethtool_queue_tx_stat_names structure */ |
| do { |
| count = 0; |
| - start = u64_stats_fetch_begin_irq(&self->stats.tx.syncp); |
| + start = u64_stats_fetch_begin(&self->stats.tx.syncp); |
| data[count] = self->stats.tx.packets; |
| data[++count] = self->stats.tx.queue_restarts; |
| - } while (u64_stats_fetch_retry_irq(&self->stats.tx.syncp, start)); |
| + } while (u64_stats_fetch_retry(&self->stats.tx.syncp, start)); |
| } |
| |
| return ++count; |
| diff --git a/drivers/net/ethernet/asix/ax88796c_main.c b/drivers/net/ethernet/asix/ax88796c_main.c |
| index 8b7cdf015a16..21376c79f671 100644 |
| --- a/drivers/net/ethernet/asix/ax88796c_main.c |
| +++ b/drivers/net/ethernet/asix/ax88796c_main.c |
| @@ -662,12 +662,12 @@ static void ax88796c_get_stats64(struct net_device *ndev, |
| s = per_cpu_ptr(ax_local->stats, cpu); |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&s->syncp); |
| + start = u64_stats_fetch_begin(&s->syncp); |
| rx_packets = u64_stats_read(&s->rx_packets); |
| rx_bytes = u64_stats_read(&s->rx_bytes); |
| tx_packets = u64_stats_read(&s->tx_packets); |
| tx_bytes = u64_stats_read(&s->tx_bytes); |
| - } while (u64_stats_fetch_retry_irq(&s->syncp, start)); |
| + } while (u64_stats_fetch_retry(&s->syncp, start)); |
| |
| stats->rx_packets += rx_packets; |
| stats->rx_bytes += rx_bytes; |
| diff --git a/drivers/net/ethernet/broadcom/b44.c b/drivers/net/ethernet/broadcom/b44.c |
| index 7f876721596c..b751dc8486dc 100644 |
| --- a/drivers/net/ethernet/broadcom/b44.c |
| +++ b/drivers/net/ethernet/broadcom/b44.c |
| @@ -1680,7 +1680,7 @@ static void b44_get_stats64(struct net_device *dev, |
| unsigned int start; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&hwstat->syncp); |
| + start = u64_stats_fetch_begin(&hwstat->syncp); |
| |
| /* Convert HW stats into rtnl_link_stats64 stats. */ |
| nstat->rx_packets = hwstat->rx_pkts; |
| @@ -1714,7 +1714,7 @@ static void b44_get_stats64(struct net_device *dev, |
| /* Carrier lost counter seems to be broken for some devices */ |
| nstat->tx_carrier_errors = hwstat->tx_carrier_lost; |
| #endif |
| - } while (u64_stats_fetch_retry_irq(&hwstat->syncp, start)); |
| + } while (u64_stats_fetch_retry(&hwstat->syncp, start)); |
| |
| } |
| |
| @@ -2082,12 +2082,12 @@ static void b44_get_ethtool_stats(struct net_device *dev, |
| do { |
| data_src = &hwstat->tx_good_octets; |
| data_dst = data; |
| - start = u64_stats_fetch_begin_irq(&hwstat->syncp); |
| + start = u64_stats_fetch_begin(&hwstat->syncp); |
| |
| for (i = 0; i < ARRAY_SIZE(b44_gstrings); i++) |
| *data_dst++ = *data_src++; |
| |
| - } while (u64_stats_fetch_retry_irq(&hwstat->syncp, start)); |
| + } while (u64_stats_fetch_retry(&hwstat->syncp, start)); |
| } |
| |
| static void b44_get_wol(struct net_device *dev, struct ethtool_wolinfo *wol) |
| diff --git a/drivers/net/ethernet/broadcom/bcmsysport.c b/drivers/net/ethernet/broadcom/bcmsysport.c |
| index 425d6ccd5413..f8b1adc389b3 100644 |
| --- a/drivers/net/ethernet/broadcom/bcmsysport.c |
| +++ b/drivers/net/ethernet/broadcom/bcmsysport.c |
| @@ -457,10 +457,10 @@ static void bcm_sysport_update_tx_stats(struct bcm_sysport_priv *priv, |
| for (q = 0; q < priv->netdev->num_tx_queues; q++) { |
| ring = &priv->tx_rings[q]; |
| do { |
| - start = u64_stats_fetch_begin_irq(&priv->syncp); |
| + start = u64_stats_fetch_begin(&priv->syncp); |
| bytes = ring->bytes; |
| packets = ring->packets; |
| - } while (u64_stats_fetch_retry_irq(&priv->syncp, start)); |
| + } while (u64_stats_fetch_retry(&priv->syncp, start)); |
| |
| *tx_bytes += bytes; |
| *tx_packets += packets; |
| @@ -504,9 +504,9 @@ static void bcm_sysport_get_stats(struct net_device *dev, |
| if (s->stat_sizeof == sizeof(u64) && |
| s->type == BCM_SYSPORT_STAT_NETDEV64) { |
| do { |
| - start = u64_stats_fetch_begin_irq(syncp); |
| + start = u64_stats_fetch_begin(syncp); |
| data[i] = *(u64 *)p; |
| - } while (u64_stats_fetch_retry_irq(syncp, start)); |
| + } while (u64_stats_fetch_retry(syncp, start)); |
| } else |
| data[i] = *(u32 *)p; |
| j++; |
| @@ -1878,10 +1878,10 @@ static void bcm_sysport_get_stats64(struct net_device *dev, |
| &stats->tx_packets); |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&priv->syncp); |
| + start = u64_stats_fetch_begin(&priv->syncp); |
| stats->rx_packets = stats64->rx_packets; |
| stats->rx_bytes = stats64->rx_bytes; |
| - } while (u64_stats_fetch_retry_irq(&priv->syncp, start)); |
| + } while (u64_stats_fetch_retry(&priv->syncp, start)); |
| } |
| |
| static void bcm_sysport_netif_start(struct net_device *dev) |
| diff --git a/drivers/net/ethernet/cortina/gemini.c b/drivers/net/ethernet/cortina/gemini.c |
| index 7c0b0bc033c9..e7137b468f5b 100644 |
| --- a/drivers/net/ethernet/cortina/gemini.c |
| +++ b/drivers/net/ethernet/cortina/gemini.c |
| @@ -1941,7 +1941,7 @@ static void gmac_get_stats64(struct net_device *netdev, |
| |
| /* Racing with RX NAPI */ |
| do { |
| - start = u64_stats_fetch_begin_irq(&port->rx_stats_syncp); |
| + start = u64_stats_fetch_begin(&port->rx_stats_syncp); |
| |
| stats->rx_packets = port->stats.rx_packets; |
| stats->rx_bytes = port->stats.rx_bytes; |
| @@ -1953,11 +1953,11 @@ static void gmac_get_stats64(struct net_device *netdev, |
| stats->rx_crc_errors = port->stats.rx_crc_errors; |
| stats->rx_frame_errors = port->stats.rx_frame_errors; |
| |
| - } while (u64_stats_fetch_retry_irq(&port->rx_stats_syncp, start)); |
| + } while (u64_stats_fetch_retry(&port->rx_stats_syncp, start)); |
| |
| /* Racing with MIB and TX completion interrupts */ |
| do { |
| - start = u64_stats_fetch_begin_irq(&port->ir_stats_syncp); |
| + start = u64_stats_fetch_begin(&port->ir_stats_syncp); |
| |
| stats->tx_errors = port->stats.tx_errors; |
| stats->tx_packets = port->stats.tx_packets; |
| @@ -1967,15 +1967,15 @@ static void gmac_get_stats64(struct net_device *netdev, |
| stats->rx_missed_errors = port->stats.rx_missed_errors; |
| stats->rx_fifo_errors = port->stats.rx_fifo_errors; |
| |
| - } while (u64_stats_fetch_retry_irq(&port->ir_stats_syncp, start)); |
| + } while (u64_stats_fetch_retry(&port->ir_stats_syncp, start)); |
| |
| /* Racing with hard_start_xmit */ |
| do { |
| - start = u64_stats_fetch_begin_irq(&port->tx_stats_syncp); |
| + start = u64_stats_fetch_begin(&port->tx_stats_syncp); |
| |
| stats->tx_dropped = port->stats.tx_dropped; |
| |
| - } while (u64_stats_fetch_retry_irq(&port->tx_stats_syncp, start)); |
| + } while (u64_stats_fetch_retry(&port->tx_stats_syncp, start)); |
| |
| stats->rx_dropped += stats->rx_missed_errors; |
| } |
| @@ -2044,18 +2044,18 @@ static void gmac_get_ethtool_stats(struct net_device *netdev, |
| /* Racing with MIB interrupt */ |
| do { |
| p = values; |
| - start = u64_stats_fetch_begin_irq(&port->ir_stats_syncp); |
| + start = u64_stats_fetch_begin(&port->ir_stats_syncp); |
| |
| for (i = 0; i < RX_STATS_NUM; i++) |
| *p++ = port->hw_stats[i]; |
| |
| - } while (u64_stats_fetch_retry_irq(&port->ir_stats_syncp, start)); |
| + } while (u64_stats_fetch_retry(&port->ir_stats_syncp, start)); |
| values = p; |
| |
| /* Racing with RX NAPI */ |
| do { |
| p = values; |
| - start = u64_stats_fetch_begin_irq(&port->rx_stats_syncp); |
| + start = u64_stats_fetch_begin(&port->rx_stats_syncp); |
| |
| for (i = 0; i < RX_STATUS_NUM; i++) |
| *p++ = port->rx_stats[i]; |
| @@ -2063,13 +2063,13 @@ static void gmac_get_ethtool_stats(struct net_device *netdev, |
| *p++ = port->rx_csum_stats[i]; |
| *p++ = port->rx_napi_exits; |
| |
| - } while (u64_stats_fetch_retry_irq(&port->rx_stats_syncp, start)); |
| + } while (u64_stats_fetch_retry(&port->rx_stats_syncp, start)); |
| values = p; |
| |
| /* Racing with TX start_xmit */ |
| do { |
| p = values; |
| - start = u64_stats_fetch_begin_irq(&port->tx_stats_syncp); |
| + start = u64_stats_fetch_begin(&port->tx_stats_syncp); |
| |
| for (i = 0; i < TX_MAX_FRAGS; i++) { |
| *values++ = port->tx_frag_stats[i]; |
| @@ -2078,7 +2078,7 @@ static void gmac_get_ethtool_stats(struct net_device *netdev, |
| *values++ = port->tx_frags_linearized; |
| *values++ = port->tx_hw_csummed; |
| |
| - } while (u64_stats_fetch_retry_irq(&port->tx_stats_syncp, start)); |
| + } while (u64_stats_fetch_retry(&port->tx_stats_syncp, start)); |
| } |
| |
| static int gmac_get_ksettings(struct net_device *netdev, |
| diff --git a/drivers/net/ethernet/emulex/benet/be_ethtool.c b/drivers/net/ethernet/emulex/benet/be_ethtool.c |
| index 77edc3d9b505..a29de29bdf23 100644 |
| --- a/drivers/net/ethernet/emulex/benet/be_ethtool.c |
| +++ b/drivers/net/ethernet/emulex/benet/be_ethtool.c |
| @@ -389,10 +389,10 @@ static void be_get_ethtool_stats(struct net_device *netdev, |
| struct be_rx_stats *stats = rx_stats(rxo); |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&stats->sync); |
| + start = u64_stats_fetch_begin(&stats->sync); |
| data[base] = stats->rx_bytes; |
| data[base + 1] = stats->rx_pkts; |
| - } while (u64_stats_fetch_retry_irq(&stats->sync, start)); |
| + } while (u64_stats_fetch_retry(&stats->sync, start)); |
| |
| for (i = 2; i < ETHTOOL_RXSTATS_NUM; i++) { |
| p = (u8 *)stats + et_rx_stats[i].offset; |
| @@ -405,19 +405,19 @@ static void be_get_ethtool_stats(struct net_device *netdev, |
| struct be_tx_stats *stats = tx_stats(txo); |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&stats->sync_compl); |
| + start = u64_stats_fetch_begin(&stats->sync_compl); |
| data[base] = stats->tx_compl; |
| - } while (u64_stats_fetch_retry_irq(&stats->sync_compl, start)); |
| + } while (u64_stats_fetch_retry(&stats->sync_compl, start)); |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&stats->sync); |
| + start = u64_stats_fetch_begin(&stats->sync); |
| for (i = 1; i < ETHTOOL_TXSTATS_NUM; i++) { |
| p = (u8 *)stats + et_tx_stats[i].offset; |
| data[base + i] = |
| (et_tx_stats[i].size == sizeof(u64)) ? |
| *(u64 *)p : *(u32 *)p; |
| } |
| - } while (u64_stats_fetch_retry_irq(&stats->sync, start)); |
| + } while (u64_stats_fetch_retry(&stats->sync, start)); |
| base += ETHTOOL_TXSTATS_NUM; |
| } |
| } |
| diff --git a/drivers/net/ethernet/emulex/benet/be_main.c b/drivers/net/ethernet/emulex/benet/be_main.c |
| index a9e4e6464a04..968fecfc03bd 100644 |
| --- a/drivers/net/ethernet/emulex/benet/be_main.c |
| +++ b/drivers/net/ethernet/emulex/benet/be_main.c |
| @@ -665,10 +665,10 @@ static void be_get_stats64(struct net_device *netdev, |
| const struct be_rx_stats *rx_stats = rx_stats(rxo); |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&rx_stats->sync); |
| + start = u64_stats_fetch_begin(&rx_stats->sync); |
| pkts = rx_stats(rxo)->rx_pkts; |
| bytes = rx_stats(rxo)->rx_bytes; |
| - } while (u64_stats_fetch_retry_irq(&rx_stats->sync, start)); |
| + } while (u64_stats_fetch_retry(&rx_stats->sync, start)); |
| stats->rx_packets += pkts; |
| stats->rx_bytes += bytes; |
| stats->multicast += rx_stats(rxo)->rx_mcast_pkts; |
| @@ -680,10 +680,10 @@ static void be_get_stats64(struct net_device *netdev, |
| const struct be_tx_stats *tx_stats = tx_stats(txo); |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&tx_stats->sync); |
| + start = u64_stats_fetch_begin(&tx_stats->sync); |
| pkts = tx_stats(txo)->tx_pkts; |
| bytes = tx_stats(txo)->tx_bytes; |
| - } while (u64_stats_fetch_retry_irq(&tx_stats->sync, start)); |
| + } while (u64_stats_fetch_retry(&tx_stats->sync, start)); |
| stats->tx_packets += pkts; |
| stats->tx_bytes += bytes; |
| } |
| @@ -2156,16 +2156,16 @@ static int be_get_new_eqd(struct be_eq_obj *eqo) |
| |
| for_all_rx_queues_on_eq(adapter, eqo, rxo, i) { |
| do { |
| - start = u64_stats_fetch_begin_irq(&rxo->stats.sync); |
| + start = u64_stats_fetch_begin(&rxo->stats.sync); |
| rx_pkts += rxo->stats.rx_pkts; |
| - } while (u64_stats_fetch_retry_irq(&rxo->stats.sync, start)); |
| + } while (u64_stats_fetch_retry(&rxo->stats.sync, start)); |
| } |
| |
| for_all_tx_queues_on_eq(adapter, eqo, txo, i) { |
| do { |
| - start = u64_stats_fetch_begin_irq(&txo->stats.sync); |
| + start = u64_stats_fetch_begin(&txo->stats.sync); |
| tx_pkts += txo->stats.tx_reqs; |
| - } while (u64_stats_fetch_retry_irq(&txo->stats.sync, start)); |
| + } while (u64_stats_fetch_retry(&txo->stats.sync, start)); |
| } |
| |
| /* Skip, if wrapped around or first calculation */ |
| diff --git a/drivers/net/ethernet/fungible/funeth/funeth_txrx.h b/drivers/net/ethernet/fungible/funeth/funeth_txrx.h |
| index 671f51135c26..53b7e95213a8 100644 |
| --- a/drivers/net/ethernet/fungible/funeth/funeth_txrx.h |
| +++ b/drivers/net/ethernet/fungible/funeth/funeth_txrx.h |
| @@ -206,9 +206,9 @@ struct funeth_rxq { |
| |
| #define FUN_QSTAT_READ(q, seq, stats_copy) \ |
| do { \ |
| - seq = u64_stats_fetch_begin_irq(&(q)->syncp); \ |
| + seq = u64_stats_fetch_begin(&(q)->syncp); \ |
| stats_copy = (q)->stats; \ |
| - } while (u64_stats_fetch_retry_irq(&(q)->syncp, (seq))) |
| + } while (u64_stats_fetch_retry(&(q)->syncp, (seq))) |
| |
| #define FUN_INT_NAME_LEN (IFNAMSIZ + 16) |
| |
| diff --git a/drivers/net/ethernet/google/gve/gve_ethtool.c b/drivers/net/ethernet/google/gve/gve_ethtool.c |
| index 033f17cb96be..0a5953089a24 100644 |
| --- a/drivers/net/ethernet/google/gve/gve_ethtool.c |
| +++ b/drivers/net/ethernet/google/gve/gve_ethtool.c |
| @@ -177,14 +177,14 @@ gve_get_ethtool_stats(struct net_device *netdev, |
| struct gve_rx_ring *rx = &priv->rx[ring]; |
| |
| start = |
| - u64_stats_fetch_begin_irq(&priv->rx[ring].statss); |
| + u64_stats_fetch_begin(&priv->rx[ring].statss); |
| tmp_rx_pkts = rx->rpackets; |
| tmp_rx_bytes = rx->rbytes; |
| tmp_rx_skb_alloc_fail = rx->rx_skb_alloc_fail; |
| tmp_rx_buf_alloc_fail = rx->rx_buf_alloc_fail; |
| tmp_rx_desc_err_dropped_pkt = |
| rx->rx_desc_err_dropped_pkt; |
| - } while (u64_stats_fetch_retry_irq(&priv->rx[ring].statss, |
| + } while (u64_stats_fetch_retry(&priv->rx[ring].statss, |
| start)); |
| rx_pkts += tmp_rx_pkts; |
| rx_bytes += tmp_rx_bytes; |
| @@ -198,10 +198,10 @@ gve_get_ethtool_stats(struct net_device *netdev, |
| if (priv->tx) { |
| do { |
| start = |
| - u64_stats_fetch_begin_irq(&priv->tx[ring].statss); |
| + u64_stats_fetch_begin(&priv->tx[ring].statss); |
| tmp_tx_pkts = priv->tx[ring].pkt_done; |
| tmp_tx_bytes = priv->tx[ring].bytes_done; |
| - } while (u64_stats_fetch_retry_irq(&priv->tx[ring].statss, |
| + } while (u64_stats_fetch_retry(&priv->tx[ring].statss, |
| start)); |
| tx_pkts += tmp_tx_pkts; |
| tx_bytes += tmp_tx_bytes; |
| @@ -259,13 +259,13 @@ gve_get_ethtool_stats(struct net_device *netdev, |
| data[i++] = rx->fill_cnt - rx->cnt; |
| do { |
| start = |
| - u64_stats_fetch_begin_irq(&priv->rx[ring].statss); |
| + u64_stats_fetch_begin(&priv->rx[ring].statss); |
| tmp_rx_bytes = rx->rbytes; |
| tmp_rx_skb_alloc_fail = rx->rx_skb_alloc_fail; |
| tmp_rx_buf_alloc_fail = rx->rx_buf_alloc_fail; |
| tmp_rx_desc_err_dropped_pkt = |
| rx->rx_desc_err_dropped_pkt; |
| - } while (u64_stats_fetch_retry_irq(&priv->rx[ring].statss, |
| + } while (u64_stats_fetch_retry(&priv->rx[ring].statss, |
| start)); |
| data[i++] = tmp_rx_bytes; |
| data[i++] = rx->rx_cont_packet_cnt; |
| @@ -331,9 +331,9 @@ gve_get_ethtool_stats(struct net_device *netdev, |
| } |
| do { |
| start = |
| - u64_stats_fetch_begin_irq(&priv->tx[ring].statss); |
| + u64_stats_fetch_begin(&priv->tx[ring].statss); |
| tmp_tx_bytes = tx->bytes_done; |
| - } while (u64_stats_fetch_retry_irq(&priv->tx[ring].statss, |
| + } while (u64_stats_fetch_retry(&priv->tx[ring].statss, |
| start)); |
| data[i++] = tmp_tx_bytes; |
| data[i++] = tx->wake_queue; |
| diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c |
| index d3f6ad586ba1..1c2cd3ee1956 100644 |
| --- a/drivers/net/ethernet/google/gve/gve_main.c |
| +++ b/drivers/net/ethernet/google/gve/gve_main.c |
| @@ -51,10 +51,10 @@ static void gve_get_stats(struct net_device *dev, struct rtnl_link_stats64 *s) |
| for (ring = 0; ring < priv->rx_cfg.num_queues; ring++) { |
| do { |
| start = |
| - u64_stats_fetch_begin_irq(&priv->rx[ring].statss); |
| + u64_stats_fetch_begin(&priv->rx[ring].statss); |
| packets = priv->rx[ring].rpackets; |
| bytes = priv->rx[ring].rbytes; |
| - } while (u64_stats_fetch_retry_irq(&priv->rx[ring].statss, |
| + } while (u64_stats_fetch_retry(&priv->rx[ring].statss, |
| start)); |
| s->rx_packets += packets; |
| s->rx_bytes += bytes; |
| @@ -64,10 +64,10 @@ static void gve_get_stats(struct net_device *dev, struct rtnl_link_stats64 *s) |
| for (ring = 0; ring < priv->tx_cfg.num_queues; ring++) { |
| do { |
| start = |
| - u64_stats_fetch_begin_irq(&priv->tx[ring].statss); |
| + u64_stats_fetch_begin(&priv->tx[ring].statss); |
| packets = priv->tx[ring].pkt_done; |
| bytes = priv->tx[ring].bytes_done; |
| - } while (u64_stats_fetch_retry_irq(&priv->tx[ring].statss, |
| + } while (u64_stats_fetch_retry(&priv->tx[ring].statss, |
| start)); |
| s->tx_packets += packets; |
| s->tx_bytes += bytes; |
| @@ -1260,9 +1260,9 @@ void gve_handle_report_stats(struct gve_priv *priv) |
| } |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&priv->tx[idx].statss); |
| + start = u64_stats_fetch_begin(&priv->tx[idx].statss); |
| tx_bytes = priv->tx[idx].bytes_done; |
| - } while (u64_stats_fetch_retry_irq(&priv->tx[idx].statss, start)); |
| + } while (u64_stats_fetch_retry(&priv->tx[idx].statss, start)); |
| stats[stats_idx++] = (struct stats) { |
| .stat_name = cpu_to_be32(TX_WAKE_CNT), |
| .value = cpu_to_be64(priv->tx[idx].wake_queue), |
| diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c |
| index 78d6752fe051..5bf81dca14fa 100644 |
| --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c |
| +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c |
| @@ -2496,7 +2496,7 @@ static void hns3_fetch_stats(struct rtnl_link_stats64 *stats, |
| unsigned int start; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&ring->syncp); |
| + start = u64_stats_fetch_begin(&ring->syncp); |
| if (is_tx) { |
| stats->tx_bytes += ring->stats.tx_bytes; |
| stats->tx_packets += ring->stats.tx_pkts; |
| @@ -2530,7 +2530,7 @@ static void hns3_fetch_stats(struct rtnl_link_stats64 *stats, |
| stats->multicast += ring->stats.rx_multicast; |
| stats->rx_length_errors += ring->stats.err_pkt_len; |
| } |
| - } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); |
| + } while (u64_stats_fetch_retry(&ring->syncp, start)); |
| } |
| |
| static void hns3_nic_get_stats64(struct net_device *netdev, |
| diff --git a/drivers/net/ethernet/huawei/hinic/hinic_rx.c b/drivers/net/ethernet/huawei/hinic/hinic_rx.c |
| index d649c6e323c8..ceec8be2a73b 100644 |
| --- a/drivers/net/ethernet/huawei/hinic/hinic_rx.c |
| +++ b/drivers/net/ethernet/huawei/hinic/hinic_rx.c |
| @@ -74,14 +74,14 @@ void hinic_rxq_get_stats(struct hinic_rxq *rxq, struct hinic_rxq_stats *stats) |
| unsigned int start; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&rxq_stats->syncp); |
| + start = u64_stats_fetch_begin(&rxq_stats->syncp); |
| stats->pkts = rxq_stats->pkts; |
| stats->bytes = rxq_stats->bytes; |
| stats->errors = rxq_stats->csum_errors + |
| rxq_stats->other_errors; |
| stats->csum_errors = rxq_stats->csum_errors; |
| stats->other_errors = rxq_stats->other_errors; |
| - } while (u64_stats_fetch_retry_irq(&rxq_stats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&rxq_stats->syncp, start)); |
| } |
| |
| /** |
| diff --git a/drivers/net/ethernet/huawei/hinic/hinic_tx.c b/drivers/net/ethernet/huawei/hinic/hinic_tx.c |
| index e91476c8ff8b..ad47ac51a139 100644 |
| --- a/drivers/net/ethernet/huawei/hinic/hinic_tx.c |
| +++ b/drivers/net/ethernet/huawei/hinic/hinic_tx.c |
| @@ -99,14 +99,14 @@ void hinic_txq_get_stats(struct hinic_txq *txq, struct hinic_txq_stats *stats) |
| unsigned int start; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&txq_stats->syncp); |
| + start = u64_stats_fetch_begin(&txq_stats->syncp); |
| stats->pkts = txq_stats->pkts; |
| stats->bytes = txq_stats->bytes; |
| stats->tx_busy = txq_stats->tx_busy; |
| stats->tx_wake = txq_stats->tx_wake; |
| stats->tx_dropped = txq_stats->tx_dropped; |
| stats->big_frags_pkts = txq_stats->big_frags_pkts; |
| - } while (u64_stats_fetch_retry_irq(&txq_stats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&txq_stats->syncp, start)); |
| } |
| |
| /** |
| diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c b/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c |
| index 2cca9e84e31e..34ab5ff9823b 100644 |
| --- a/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c |
| +++ b/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c |
| @@ -1229,10 +1229,10 @@ static void fm10k_get_stats64(struct net_device *netdev, |
| continue; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&ring->syncp); |
| + start = u64_stats_fetch_begin(&ring->syncp); |
| packets = ring->stats.packets; |
| bytes = ring->stats.bytes; |
| - } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); |
| + } while (u64_stats_fetch_retry(&ring->syncp, start)); |
| |
| stats->rx_packets += packets; |
| stats->rx_bytes += bytes; |
| @@ -1245,10 +1245,10 @@ static void fm10k_get_stats64(struct net_device *netdev, |
| continue; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&ring->syncp); |
| + start = u64_stats_fetch_begin(&ring->syncp); |
| packets = ring->stats.packets; |
| bytes = ring->stats.bytes; |
| - } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); |
| + } while (u64_stats_fetch_retry(&ring->syncp, start)); |
| |
| stats->tx_packets += packets; |
| stats->tx_bytes += bytes; |
| diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c |
| index 107bcca7db8c..8f36fe90180f 100644 |
| --- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c |
| +++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c |
| @@ -154,7 +154,7 @@ __i40e_add_ethtool_stats(u64 **data, void *pointer, |
| * @ring: the ring to copy |
| * |
| * Queue statistics must be copied while protected by |
| - * u64_stats_fetch_begin_irq, so we can't directly use i40e_add_ethtool_stats. |
| + * u64_stats_fetch_begin, so we can't directly use i40e_add_ethtool_stats. |
| * Assumes that queue stats are defined in i40e_gstrings_queue_stats. If the |
| * ring pointer is null, zero out the queue stat values and update the data |
| * pointer. Otherwise safely copy the stats from the ring into the supplied |
| @@ -172,16 +172,16 @@ i40e_add_queue_stats(u64 **data, struct i40e_ring *ring) |
| |
| /* To avoid invalid statistics values, ensure that we keep retrying |
| * the copy until we get a consistent value according to |
| - * u64_stats_fetch_retry_irq. But first, make sure our ring is |
| + * u64_stats_fetch_retry. But first, make sure our ring is |
| * non-null before attempting to access its syncp. |
| */ |
| do { |
| - start = !ring ? 0 : u64_stats_fetch_begin_irq(&ring->syncp); |
| + start = !ring ? 0 : u64_stats_fetch_begin(&ring->syncp); |
| for (i = 0; i < size; i++) { |
| i40e_add_one_ethtool_stat(&(*data)[i], ring, |
| &stats[i]); |
| } |
| - } while (ring && u64_stats_fetch_retry_irq(&ring->syncp, start)); |
| + } while (ring && u64_stats_fetch_retry(&ring->syncp, start)); |
| |
| /* Once we successfully copy the stats in, update the data pointer */ |
| *data += size; |
| diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c |
| index d8a7fb21b7b7..845c9518f4e4 100644 |
| --- a/drivers/net/ethernet/intel/i40e/i40e_main.c |
| +++ b/drivers/net/ethernet/intel/i40e/i40e_main.c |
| @@ -425,10 +425,10 @@ static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring, |
| unsigned int start; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&ring->syncp); |
| + start = u64_stats_fetch_begin(&ring->syncp); |
| packets = ring->stats.packets; |
| bytes = ring->stats.bytes; |
| - } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); |
| + } while (u64_stats_fetch_retry(&ring->syncp, start)); |
| |
| stats->tx_packets += packets; |
| stats->tx_bytes += bytes; |
| @@ -478,10 +478,10 @@ static void i40e_get_netdev_stats_struct(struct net_device *netdev, |
| if (!ring) |
| continue; |
| do { |
| - start = u64_stats_fetch_begin_irq(&ring->syncp); |
| + start = u64_stats_fetch_begin(&ring->syncp); |
| packets = ring->stats.packets; |
| bytes = ring->stats.bytes; |
| - } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); |
| + } while (u64_stats_fetch_retry(&ring->syncp, start)); |
| |
| stats->rx_packets += packets; |
| stats->rx_bytes += bytes; |
| @@ -903,10 +903,10 @@ static void i40e_update_vsi_stats(struct i40e_vsi *vsi) |
| continue; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&p->syncp); |
| + start = u64_stats_fetch_begin(&p->syncp); |
| packets = p->stats.packets; |
| bytes = p->stats.bytes; |
| - } while (u64_stats_fetch_retry_irq(&p->syncp, start)); |
| + } while (u64_stats_fetch_retry(&p->syncp, start)); |
| tx_b += bytes; |
| tx_p += packets; |
| tx_restart += p->tx_stats.restart_queue; |
| @@ -921,10 +921,10 @@ static void i40e_update_vsi_stats(struct i40e_vsi *vsi) |
| continue; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&p->syncp); |
| + start = u64_stats_fetch_begin(&p->syncp); |
| packets = p->stats.packets; |
| bytes = p->stats.bytes; |
| - } while (u64_stats_fetch_retry_irq(&p->syncp, start)); |
| + } while (u64_stats_fetch_retry(&p->syncp, start)); |
| rx_b += bytes; |
| rx_p += packets; |
| rx_buf += p->rx_stats.alloc_buff_failed; |
| @@ -941,10 +941,10 @@ static void i40e_update_vsi_stats(struct i40e_vsi *vsi) |
| continue; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&p->syncp); |
| + start = u64_stats_fetch_begin(&p->syncp); |
| packets = p->stats.packets; |
| bytes = p->stats.bytes; |
| - } while (u64_stats_fetch_retry_irq(&p->syncp, start)); |
| + } while (u64_stats_fetch_retry(&p->syncp, start)); |
| tx_b += bytes; |
| tx_p += packets; |
| tx_restart += p->tx_stats.restart_queue; |
| diff --git a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c |
| index f4ac2b164b3e..892c6a4f03bb 100644 |
| --- a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c |
| +++ b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c |
| @@ -147,7 +147,7 @@ __iavf_add_ethtool_stats(u64 **data, void *pointer, |
| * @ring: the ring to copy |
| * |
| * Queue statistics must be copied while protected by |
| - * u64_stats_fetch_begin_irq, so we can't directly use iavf_add_ethtool_stats. |
| + * u64_stats_fetch_begin, so we can't directly use iavf_add_ethtool_stats. |
| * Assumes that queue stats are defined in iavf_gstrings_queue_stats. If the |
| * ring pointer is null, zero out the queue stat values and update the data |
| * pointer. Otherwise safely copy the stats from the ring into the supplied |
| @@ -165,14 +165,14 @@ iavf_add_queue_stats(u64 **data, struct iavf_ring *ring) |
| |
| /* To avoid invalid statistics values, ensure that we keep retrying |
| * the copy until we get a consistent value according to |
| - * u64_stats_fetch_retry_irq. But first, make sure our ring is |
| + * u64_stats_fetch_retry. But first, make sure our ring is |
| * non-null before attempting to access its syncp. |
| */ |
| do { |
| - start = !ring ? 0 : u64_stats_fetch_begin_irq(&ring->syncp); |
| + start = !ring ? 0 : u64_stats_fetch_begin(&ring->syncp); |
| for (i = 0; i < size; i++) |
| iavf_add_one_ethtool_stat(&(*data)[i], ring, &stats[i]); |
| - } while (ring && u64_stats_fetch_retry_irq(&ring->syncp, start)); |
| + } while (ring && u64_stats_fetch_retry(&ring->syncp, start)); |
| |
| /* Once we successfully copy the stats in, update the data pointer */ |
| *data += size; |
| diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c |
| index 3117f65253b3..103a0ed66962 100644 |
| --- a/drivers/net/ethernet/intel/ice/ice_main.c |
| +++ b/drivers/net/ethernet/intel/ice/ice_main.c |
| @@ -6419,10 +6419,10 @@ ice_fetch_u64_stats_per_ring(struct u64_stats_sync *syncp, |
| unsigned int start; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(syncp); |
| + start = u64_stats_fetch_begin(syncp); |
| *pkts = stats.pkts; |
| *bytes = stats.bytes; |
| - } while (u64_stats_fetch_retry_irq(syncp, start)); |
| + } while (u64_stats_fetch_retry(syncp, start)); |
| } |
| |
| /** |
| diff --git a/drivers/net/ethernet/intel/igb/igb_ethtool.c b/drivers/net/ethernet/intel/igb/igb_ethtool.c |
| index ceff537d9d22..4ee849985e2b 100644 |
| --- a/drivers/net/ethernet/intel/igb/igb_ethtool.c |
| +++ b/drivers/net/ethernet/intel/igb/igb_ethtool.c |
| @@ -2316,15 +2316,15 @@ static void igb_get_ethtool_stats(struct net_device *netdev, |
| |
| ring = adapter->tx_ring[j]; |
| do { |
| - start = u64_stats_fetch_begin_irq(&ring->tx_syncp); |
| + start = u64_stats_fetch_begin(&ring->tx_syncp); |
| data[i] = ring->tx_stats.packets; |
| data[i+1] = ring->tx_stats.bytes; |
| data[i+2] = ring->tx_stats.restart_queue; |
| - } while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start)); |
| + } while (u64_stats_fetch_retry(&ring->tx_syncp, start)); |
| do { |
| - start = u64_stats_fetch_begin_irq(&ring->tx_syncp2); |
| + start = u64_stats_fetch_begin(&ring->tx_syncp2); |
| restart2 = ring->tx_stats.restart_queue2; |
| - } while (u64_stats_fetch_retry_irq(&ring->tx_syncp2, start)); |
| + } while (u64_stats_fetch_retry(&ring->tx_syncp2, start)); |
| data[i+2] += restart2; |
| |
| i += IGB_TX_QUEUE_STATS_LEN; |
| @@ -2332,13 +2332,13 @@ static void igb_get_ethtool_stats(struct net_device *netdev, |
| for (j = 0; j < adapter->num_rx_queues; j++) { |
| ring = adapter->rx_ring[j]; |
| do { |
| - start = u64_stats_fetch_begin_irq(&ring->rx_syncp); |
| + start = u64_stats_fetch_begin(&ring->rx_syncp); |
| data[i] = ring->rx_stats.packets; |
| data[i+1] = ring->rx_stats.bytes; |
| data[i+2] = ring->rx_stats.drops; |
| data[i+3] = ring->rx_stats.csum_err; |
| data[i+4] = ring->rx_stats.alloc_failed; |
| - } while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start)); |
| + } while (u64_stats_fetch_retry(&ring->rx_syncp, start)); |
| i += IGB_RX_QUEUE_STATS_LEN; |
| } |
| spin_unlock(&adapter->stats64_lock); |
| diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c |
| index 81d9a5338be5..bea1d2b24214 100644 |
| --- a/drivers/net/ethernet/intel/igb/igb_main.c |
| +++ b/drivers/net/ethernet/intel/igb/igb_main.c |
| @@ -6660,10 +6660,10 @@ void igb_update_stats(struct igb_adapter *adapter) |
| } |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&ring->rx_syncp); |
| + start = u64_stats_fetch_begin(&ring->rx_syncp); |
| _bytes = ring->rx_stats.bytes; |
| _packets = ring->rx_stats.packets; |
| - } while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start)); |
| + } while (u64_stats_fetch_retry(&ring->rx_syncp, start)); |
| bytes += _bytes; |
| packets += _packets; |
| } |
| @@ -6676,10 +6676,10 @@ void igb_update_stats(struct igb_adapter *adapter) |
| for (i = 0; i < adapter->num_tx_queues; i++) { |
| struct igb_ring *ring = adapter->tx_ring[i]; |
| do { |
| - start = u64_stats_fetch_begin_irq(&ring->tx_syncp); |
| + start = u64_stats_fetch_begin(&ring->tx_syncp); |
| _bytes = ring->tx_stats.bytes; |
| _packets = ring->tx_stats.packets; |
| - } while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start)); |
| + } while (u64_stats_fetch_retry(&ring->tx_syncp, start)); |
| bytes += _bytes; |
| packets += _packets; |
| } |
| diff --git a/drivers/net/ethernet/intel/igc/igc_ethtool.c b/drivers/net/ethernet/intel/igc/igc_ethtool.c |
| index 2bee9cace598..f7284fa4324a 100644 |
| --- a/drivers/net/ethernet/intel/igc/igc_ethtool.c |
| +++ b/drivers/net/ethernet/intel/igc/igc_ethtool.c |
| @@ -840,15 +840,15 @@ static void igc_ethtool_get_stats(struct net_device *netdev, |
| |
| ring = adapter->tx_ring[j]; |
| do { |
| - start = u64_stats_fetch_begin_irq(&ring->tx_syncp); |
| + start = u64_stats_fetch_begin(&ring->tx_syncp); |
| data[i] = ring->tx_stats.packets; |
| data[i + 1] = ring->tx_stats.bytes; |
| data[i + 2] = ring->tx_stats.restart_queue; |
| - } while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start)); |
| + } while (u64_stats_fetch_retry(&ring->tx_syncp, start)); |
| do { |
| - start = u64_stats_fetch_begin_irq(&ring->tx_syncp2); |
| + start = u64_stats_fetch_begin(&ring->tx_syncp2); |
| restart2 = ring->tx_stats.restart_queue2; |
| - } while (u64_stats_fetch_retry_irq(&ring->tx_syncp2, start)); |
| + } while (u64_stats_fetch_retry(&ring->tx_syncp2, start)); |
| data[i + 2] += restart2; |
| |
| i += IGC_TX_QUEUE_STATS_LEN; |
| @@ -856,13 +856,13 @@ static void igc_ethtool_get_stats(struct net_device *netdev, |
| for (j = 0; j < adapter->num_rx_queues; j++) { |
| ring = adapter->rx_ring[j]; |
| do { |
| - start = u64_stats_fetch_begin_irq(&ring->rx_syncp); |
| + start = u64_stats_fetch_begin(&ring->rx_syncp); |
| data[i] = ring->rx_stats.packets; |
| data[i + 1] = ring->rx_stats.bytes; |
| data[i + 2] = ring->rx_stats.drops; |
| data[i + 3] = ring->rx_stats.csum_err; |
| data[i + 4] = ring->rx_stats.alloc_failed; |
| - } while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start)); |
| + } while (u64_stats_fetch_retry(&ring->rx_syncp, start)); |
| i += IGC_RX_QUEUE_STATS_LEN; |
| } |
| spin_unlock(&adapter->stats64_lock); |
| diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c |
| index e052f49cc08d..9f2851afbd81 100644 |
| --- a/drivers/net/ethernet/intel/igc/igc_main.c |
| +++ b/drivers/net/ethernet/intel/igc/igc_main.c |
| @@ -4868,10 +4868,10 @@ void igc_update_stats(struct igc_adapter *adapter) |
| } |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&ring->rx_syncp); |
| + start = u64_stats_fetch_begin(&ring->rx_syncp); |
| _bytes = ring->rx_stats.bytes; |
| _packets = ring->rx_stats.packets; |
| - } while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start)); |
| + } while (u64_stats_fetch_retry(&ring->rx_syncp, start)); |
| bytes += _bytes; |
| packets += _packets; |
| } |
| @@ -4885,10 +4885,10 @@ void igc_update_stats(struct igc_adapter *adapter) |
| struct igc_ring *ring = adapter->tx_ring[i]; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&ring->tx_syncp); |
| + start = u64_stats_fetch_begin(&ring->tx_syncp); |
| _bytes = ring->tx_stats.bytes; |
| _packets = ring->tx_stats.packets; |
| - } while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start)); |
| + } while (u64_stats_fetch_retry(&ring->tx_syncp, start)); |
| bytes += _bytes; |
| packets += _packets; |
| } |
| diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c |
| index f8e65e18284e..80e1003e9626 100644 |
| --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c |
| +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c |
| @@ -1335,10 +1335,10 @@ static void ixgbe_get_ethtool_stats(struct net_device *netdev, |
| } |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&ring->syncp); |
| + start = u64_stats_fetch_begin(&ring->syncp); |
| data[i] = ring->stats.packets; |
| data[i+1] = ring->stats.bytes; |
| - } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); |
| + } while (u64_stats_fetch_retry(&ring->syncp, start)); |
| i += 2; |
| } |
| for (j = 0; j < IXGBE_NUM_RX_QUEUES; j++) { |
| @@ -1351,10 +1351,10 @@ static void ixgbe_get_ethtool_stats(struct net_device *netdev, |
| } |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&ring->syncp); |
| + start = u64_stats_fetch_begin(&ring->syncp); |
| data[i] = ring->stats.packets; |
| data[i+1] = ring->stats.bytes; |
| - } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); |
| + } while (u64_stats_fetch_retry(&ring->syncp, start)); |
| i += 2; |
| } |
| |
| diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c |
| index 086cc2573033..f48de0bca8a4 100644 |
| --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c |
| +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c |
| @@ -9047,10 +9047,10 @@ static void ixgbe_get_ring_stats64(struct rtnl_link_stats64 *stats, |
| |
| if (ring) { |
| do { |
| - start = u64_stats_fetch_begin_irq(&ring->syncp); |
| + start = u64_stats_fetch_begin(&ring->syncp); |
| packets = ring->stats.packets; |
| bytes = ring->stats.bytes; |
| - } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); |
| + } while (u64_stats_fetch_retry(&ring->syncp, start)); |
| stats->tx_packets += packets; |
| stats->tx_bytes += bytes; |
| } |
| @@ -9070,10 +9070,10 @@ static void ixgbe_get_stats64(struct net_device *netdev, |
| |
| if (ring) { |
| do { |
| - start = u64_stats_fetch_begin_irq(&ring->syncp); |
| + start = u64_stats_fetch_begin(&ring->syncp); |
| packets = ring->stats.packets; |
| bytes = ring->stats.bytes; |
| - } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); |
| + } while (u64_stats_fetch_retry(&ring->syncp, start)); |
| stats->rx_packets += packets; |
| stats->rx_bytes += bytes; |
| } |
| diff --git a/drivers/net/ethernet/intel/ixgbevf/ethtool.c b/drivers/net/ethernet/intel/ixgbevf/ethtool.c |
| index ccfa6b91aac6..296915414a7c 100644 |
| --- a/drivers/net/ethernet/intel/ixgbevf/ethtool.c |
| +++ b/drivers/net/ethernet/intel/ixgbevf/ethtool.c |
| @@ -458,10 +458,10 @@ static void ixgbevf_get_ethtool_stats(struct net_device *netdev, |
| } |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&ring->syncp); |
| + start = u64_stats_fetch_begin(&ring->syncp); |
| data[i] = ring->stats.packets; |
| data[i + 1] = ring->stats.bytes; |
| - } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); |
| + } while (u64_stats_fetch_retry(&ring->syncp, start)); |
| i += 2; |
| } |
| |
| @@ -475,10 +475,10 @@ static void ixgbevf_get_ethtool_stats(struct net_device *netdev, |
| } |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&ring->syncp); |
| + start = u64_stats_fetch_begin(&ring->syncp); |
| data[i] = ring->stats.packets; |
| data[i + 1] = ring->stats.bytes; |
| - } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); |
| + } while (u64_stats_fetch_retry(&ring->syncp, start)); |
| i += 2; |
| } |
| |
| @@ -492,10 +492,10 @@ static void ixgbevf_get_ethtool_stats(struct net_device *netdev, |
| } |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&ring->syncp); |
| + start = u64_stats_fetch_begin(&ring->syncp); |
| data[i] = ring->stats.packets; |
| data[i + 1] = ring->stats.bytes; |
| - } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); |
| + } while (u64_stats_fetch_retry(&ring->syncp, start)); |
| i += 2; |
| } |
| } |
| diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c |
| index e338fa572793..a9479ddf68eb 100644 |
| --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c |
| +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c |
| @@ -4350,10 +4350,10 @@ static void ixgbevf_get_tx_ring_stats(struct rtnl_link_stats64 *stats, |
| |
| if (ring) { |
| do { |
| - start = u64_stats_fetch_begin_irq(&ring->syncp); |
| + start = u64_stats_fetch_begin(&ring->syncp); |
| bytes = ring->stats.bytes; |
| packets = ring->stats.packets; |
| - } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); |
| + } while (u64_stats_fetch_retry(&ring->syncp, start)); |
| stats->tx_bytes += bytes; |
| stats->tx_packets += packets; |
| } |
| @@ -4376,10 +4376,10 @@ static void ixgbevf_get_stats(struct net_device *netdev, |
| for (i = 0; i < adapter->num_rx_queues; i++) { |
| ring = adapter->rx_ring[i]; |
| do { |
| - start = u64_stats_fetch_begin_irq(&ring->syncp); |
| + start = u64_stats_fetch_begin(&ring->syncp); |
| bytes = ring->stats.bytes; |
| packets = ring->stats.packets; |
| - } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); |
| + } while (u64_stats_fetch_retry(&ring->syncp, start)); |
| stats->rx_bytes += bytes; |
| stats->rx_packets += packets; |
| } |
| diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c |
| index eb4ebaa1c92f..327f03f80836 100644 |
| --- a/drivers/net/ethernet/marvell/mvneta.c |
| +++ b/drivers/net/ethernet/marvell/mvneta.c |
| @@ -813,14 +813,14 @@ mvneta_get_stats64(struct net_device *dev, |
| |
| cpu_stats = per_cpu_ptr(pp->stats, cpu); |
| do { |
| - start = u64_stats_fetch_begin_irq(&cpu_stats->syncp); |
| + start = u64_stats_fetch_begin(&cpu_stats->syncp); |
| rx_packets = cpu_stats->es.ps.rx_packets; |
| rx_bytes = cpu_stats->es.ps.rx_bytes; |
| rx_dropped = cpu_stats->rx_dropped; |
| rx_errors = cpu_stats->rx_errors; |
| tx_packets = cpu_stats->es.ps.tx_packets; |
| tx_bytes = cpu_stats->es.ps.tx_bytes; |
| - } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&cpu_stats->syncp, start)); |
| |
| stats->rx_packets += rx_packets; |
| stats->rx_bytes += rx_bytes; |
| @@ -4765,7 +4765,7 @@ mvneta_ethtool_update_pcpu_stats(struct mvneta_port *pp, |
| |
| stats = per_cpu_ptr(pp->stats, cpu); |
| do { |
| - start = u64_stats_fetch_begin_irq(&stats->syncp); |
| + start = u64_stats_fetch_begin(&stats->syncp); |
| skb_alloc_error = stats->es.skb_alloc_error; |
| refill_error = stats->es.refill_error; |
| xdp_redirect = stats->es.ps.xdp_redirect; |
| @@ -4775,7 +4775,7 @@ mvneta_ethtool_update_pcpu_stats(struct mvneta_port *pp, |
| xdp_xmit_err = stats->es.ps.xdp_xmit_err; |
| xdp_tx = stats->es.ps.xdp_tx; |
| xdp_tx_err = stats->es.ps.xdp_tx_err; |
| - } while (u64_stats_fetch_retry_irq(&stats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&stats->syncp, start)); |
| |
| es->skb_alloc_error += skb_alloc_error; |
| es->refill_error += refill_error; |
| diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c |
| index 2f80ee84c7ec..e2037c55a10f 100644 |
| --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c |
| +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c |
| @@ -2033,7 +2033,7 @@ mvpp2_get_xdp_stats(struct mvpp2_port *port, struct mvpp2_pcpu_stats *xdp_stats) |
| |
| cpu_stats = per_cpu_ptr(port->stats, cpu); |
| do { |
| - start = u64_stats_fetch_begin_irq(&cpu_stats->syncp); |
| + start = u64_stats_fetch_begin(&cpu_stats->syncp); |
| xdp_redirect = cpu_stats->xdp_redirect; |
| xdp_pass = cpu_stats->xdp_pass; |
| xdp_drop = cpu_stats->xdp_drop; |
| @@ -2041,7 +2041,7 @@ mvpp2_get_xdp_stats(struct mvpp2_port *port, struct mvpp2_pcpu_stats *xdp_stats) |
| xdp_xmit_err = cpu_stats->xdp_xmit_err; |
| xdp_tx = cpu_stats->xdp_tx; |
| xdp_tx_err = cpu_stats->xdp_tx_err; |
| - } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&cpu_stats->syncp, start)); |
| |
| xdp_stats->xdp_redirect += xdp_redirect; |
| xdp_stats->xdp_pass += xdp_pass; |
| @@ -5140,12 +5140,12 @@ mvpp2_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) |
| |
| cpu_stats = per_cpu_ptr(port->stats, cpu); |
| do { |
| - start = u64_stats_fetch_begin_irq(&cpu_stats->syncp); |
| + start = u64_stats_fetch_begin(&cpu_stats->syncp); |
| rx_packets = cpu_stats->rx_packets; |
| rx_bytes = cpu_stats->rx_bytes; |
| tx_packets = cpu_stats->tx_packets; |
| tx_bytes = cpu_stats->tx_bytes; |
| - } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&cpu_stats->syncp, start)); |
| |
| stats->rx_packets += rx_packets; |
| stats->rx_bytes += rx_bytes; |
| diff --git a/drivers/net/ethernet/marvell/sky2.c b/drivers/net/ethernet/marvell/sky2.c |
| index ab33ba1c3023..ff97b140886a 100644 |
| --- a/drivers/net/ethernet/marvell/sky2.c |
| +++ b/drivers/net/ethernet/marvell/sky2.c |
| @@ -3894,19 +3894,19 @@ static void sky2_get_stats(struct net_device *dev, |
| u64 _bytes, _packets; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&sky2->rx_stats.syncp); |
| + start = u64_stats_fetch_begin(&sky2->rx_stats.syncp); |
| _bytes = sky2->rx_stats.bytes; |
| _packets = sky2->rx_stats.packets; |
| - } while (u64_stats_fetch_retry_irq(&sky2->rx_stats.syncp, start)); |
| + } while (u64_stats_fetch_retry(&sky2->rx_stats.syncp, start)); |
| |
| stats->rx_packets = _packets; |
| stats->rx_bytes = _bytes; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&sky2->tx_stats.syncp); |
| + start = u64_stats_fetch_begin(&sky2->tx_stats.syncp); |
| _bytes = sky2->tx_stats.bytes; |
| _packets = sky2->tx_stats.packets; |
| - } while (u64_stats_fetch_retry_irq(&sky2->tx_stats.syncp, start)); |
| + } while (u64_stats_fetch_retry(&sky2->tx_stats.syncp, start)); |
| |
| stats->tx_packets = _packets; |
| stats->tx_bytes = _bytes; |
| diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c |
| index fecf3dd22dfa..62726113cba7 100644 |
| --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c |
| +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c |
| @@ -865,7 +865,7 @@ static void mtk_get_stats64(struct net_device *dev, |
| } |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&hw_stats->syncp); |
| + start = u64_stats_fetch_begin(&hw_stats->syncp); |
| storage->rx_packets = hw_stats->rx_packets; |
| storage->tx_packets = hw_stats->tx_packets; |
| storage->rx_bytes = hw_stats->rx_bytes; |
| @@ -877,7 +877,7 @@ static void mtk_get_stats64(struct net_device *dev, |
| storage->rx_crc_errors = hw_stats->rx_fcs_errors; |
| storage->rx_errors = hw_stats->rx_checksum_errors; |
| storage->tx_aborted_errors = hw_stats->tx_skip; |
| - } while (u64_stats_fetch_retry_irq(&hw_stats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&hw_stats->syncp, start)); |
| |
| storage->tx_errors = dev->stats.tx_errors; |
| storage->rx_dropped = dev->stats.rx_dropped; |
| @@ -3693,13 +3693,13 @@ static void mtk_get_ethtool_stats(struct net_device *dev, |
| |
| do { |
| data_dst = data; |
| - start = u64_stats_fetch_begin_irq(&hwstats->syncp); |
| + start = u64_stats_fetch_begin(&hwstats->syncp); |
| |
| for (i = 0; i < ARRAY_SIZE(mtk_ethtool_stats); i++) |
| *data_dst++ = *(data_src + mtk_ethtool_stats[i].offset); |
| if (mtk_page_pool_enabled(mac->hw)) |
| mtk_ethtool_pp_stats(mac->hw, data_dst); |
| - } while (u64_stats_fetch_retry_irq(&hwstats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&hwstats->syncp, start)); |
| } |
| |
| static int mtk_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd, |
| diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c |
| index 67ecdb9e708f..8345499563a4 100644 |
| --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c |
| +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c |
| @@ -827,12 +827,12 @@ mlxsw_sp_port_get_sw_stats64(const struct net_device *dev, |
| for_each_possible_cpu(i) { |
| p = per_cpu_ptr(mlxsw_sp_port->pcpu_stats, i); |
| do { |
| - start = u64_stats_fetch_begin_irq(&p->syncp); |
| + start = u64_stats_fetch_begin(&p->syncp); |
| rx_packets = p->rx_packets; |
| rx_bytes = p->rx_bytes; |
| tx_packets = p->tx_packets; |
| tx_bytes = p->tx_bytes; |
| - } while (u64_stats_fetch_retry_irq(&p->syncp, start)); |
| + } while (u64_stats_fetch_retry(&p->syncp, start)); |
| |
| stats->rx_packets += rx_packets; |
| stats->rx_bytes += rx_bytes; |
| diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c |
| index b751b03eddfb..14d17c6f90ff 100644 |
| --- a/drivers/net/ethernet/microsoft/mana/mana_en.c |
| +++ b/drivers/net/ethernet/microsoft/mana/mana_en.c |
| @@ -316,10 +316,10 @@ static void mana_get_stats64(struct net_device *ndev, |
| rx_stats = &apc->rxqs[q]->stats; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&rx_stats->syncp); |
| + start = u64_stats_fetch_begin(&rx_stats->syncp); |
| packets = rx_stats->packets; |
| bytes = rx_stats->bytes; |
| - } while (u64_stats_fetch_retry_irq(&rx_stats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&rx_stats->syncp, start)); |
| |
| st->rx_packets += packets; |
| st->rx_bytes += bytes; |
| @@ -329,10 +329,10 @@ static void mana_get_stats64(struct net_device *ndev, |
| tx_stats = &apc->tx_qp[q].txq.stats; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&tx_stats->syncp); |
| + start = u64_stats_fetch_begin(&tx_stats->syncp); |
| packets = tx_stats->packets; |
| bytes = tx_stats->bytes; |
| - } while (u64_stats_fetch_retry_irq(&tx_stats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&tx_stats->syncp, start)); |
| |
| st->tx_packets += packets; |
| st->tx_bytes += bytes; |
| diff --git a/drivers/net/ethernet/microsoft/mana/mana_ethtool.c b/drivers/net/ethernet/microsoft/mana/mana_ethtool.c |
| index c530db76880f..96d55c91c969 100644 |
| --- a/drivers/net/ethernet/microsoft/mana/mana_ethtool.c |
| +++ b/drivers/net/ethernet/microsoft/mana/mana_ethtool.c |
| @@ -90,13 +90,13 @@ static void mana_get_ethtool_stats(struct net_device *ndev, |
| rx_stats = &apc->rxqs[q]->stats; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&rx_stats->syncp); |
| + start = u64_stats_fetch_begin(&rx_stats->syncp); |
| packets = rx_stats->packets; |
| bytes = rx_stats->bytes; |
| xdp_drop = rx_stats->xdp_drop; |
| xdp_tx = rx_stats->xdp_tx; |
| xdp_redirect = rx_stats->xdp_redirect; |
| - } while (u64_stats_fetch_retry_irq(&rx_stats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&rx_stats->syncp, start)); |
| |
| data[i++] = packets; |
| data[i++] = bytes; |
| @@ -109,11 +109,11 @@ static void mana_get_ethtool_stats(struct net_device *ndev, |
| tx_stats = &apc->tx_qp[q].txq.stats; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&tx_stats->syncp); |
| + start = u64_stats_fetch_begin(&tx_stats->syncp); |
| packets = tx_stats->packets; |
| bytes = tx_stats->bytes; |
| xdp_xmit = tx_stats->xdp_xmit; |
| - } while (u64_stats_fetch_retry_irq(&tx_stats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&tx_stats->syncp, start)); |
| |
| data[i++] = packets; |
| data[i++] = bytes; |
| diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c |
| index 27f4786ace4f..a5ca5c4a7896 100644 |
| --- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c |
| +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c |
| @@ -1631,21 +1631,21 @@ static void nfp_net_stat64(struct net_device *netdev, |
| unsigned int start; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&r_vec->rx_sync); |
| + start = u64_stats_fetch_begin(&r_vec->rx_sync); |
| data[0] = r_vec->rx_pkts; |
| data[1] = r_vec->rx_bytes; |
| data[2] = r_vec->rx_drops; |
| - } while (u64_stats_fetch_retry_irq(&r_vec->rx_sync, start)); |
| + } while (u64_stats_fetch_retry(&r_vec->rx_sync, start)); |
| stats->rx_packets += data[0]; |
| stats->rx_bytes += data[1]; |
| stats->rx_dropped += data[2]; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&r_vec->tx_sync); |
| + start = u64_stats_fetch_begin(&r_vec->tx_sync); |
| data[0] = r_vec->tx_pkts; |
| data[1] = r_vec->tx_bytes; |
| data[2] = r_vec->tx_errors; |
| - } while (u64_stats_fetch_retry_irq(&r_vec->tx_sync, start)); |
| + } while (u64_stats_fetch_retry(&r_vec->tx_sync, start)); |
| stats->tx_packets += data[0]; |
| stats->tx_bytes += data[1]; |
| stats->tx_errors += data[2]; |
| diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c |
| index af376b900067..cc97b3d00414 100644 |
| --- a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c |
| +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c |
| @@ -881,7 +881,7 @@ static u64 *nfp_vnic_get_sw_stats(struct net_device *netdev, u64 *data) |
| unsigned int start; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&nn->r_vecs[i].rx_sync); |
| + start = u64_stats_fetch_begin(&nn->r_vecs[i].rx_sync); |
| data[0] = nn->r_vecs[i].rx_pkts; |
| tmp[0] = nn->r_vecs[i].hw_csum_rx_ok; |
| tmp[1] = nn->r_vecs[i].hw_csum_rx_inner_ok; |
| @@ -889,10 +889,10 @@ static u64 *nfp_vnic_get_sw_stats(struct net_device *netdev, u64 *data) |
| tmp[3] = nn->r_vecs[i].hw_csum_rx_error; |
| tmp[4] = nn->r_vecs[i].rx_replace_buf_alloc_fail; |
| tmp[5] = nn->r_vecs[i].hw_tls_rx; |
| - } while (u64_stats_fetch_retry_irq(&nn->r_vecs[i].rx_sync, start)); |
| + } while (u64_stats_fetch_retry(&nn->r_vecs[i].rx_sync, start)); |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&nn->r_vecs[i].tx_sync); |
| + start = u64_stats_fetch_begin(&nn->r_vecs[i].tx_sync); |
| data[1] = nn->r_vecs[i].tx_pkts; |
| data[2] = nn->r_vecs[i].tx_busy; |
| tmp[6] = nn->r_vecs[i].hw_csum_tx; |
| @@ -902,7 +902,7 @@ static u64 *nfp_vnic_get_sw_stats(struct net_device *netdev, u64 *data) |
| tmp[10] = nn->r_vecs[i].hw_tls_tx; |
| tmp[11] = nn->r_vecs[i].tls_tx_fallback; |
| tmp[12] = nn->r_vecs[i].tls_tx_no_fallback; |
| - } while (u64_stats_fetch_retry_irq(&nn->r_vecs[i].tx_sync, start)); |
| + } while (u64_stats_fetch_retry(&nn->r_vecs[i].tx_sync, start)); |
| |
| data += NN_RVEC_PER_Q_STATS; |
| |
| diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c |
| index 8b77582bdfa0..a6b6ca1fd55e 100644 |
| --- a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c |
| +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c |
| @@ -134,13 +134,13 @@ nfp_repr_get_host_stats64(const struct net_device *netdev, |
| |
| repr_stats = per_cpu_ptr(repr->stats, i); |
| do { |
| - start = u64_stats_fetch_begin_irq(&repr_stats->syncp); |
| + start = u64_stats_fetch_begin(&repr_stats->syncp); |
| tbytes = repr_stats->tx_bytes; |
| tpkts = repr_stats->tx_packets; |
| tdrops = repr_stats->tx_drops; |
| rbytes = repr_stats->rx_bytes; |
| rpkts = repr_stats->rx_packets; |
| - } while (u64_stats_fetch_retry_irq(&repr_stats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&repr_stats->syncp, start)); |
| |
| stats->tx_bytes += tbytes; |
| stats->tx_packets += tpkts; |
| diff --git a/drivers/net/ethernet/nvidia/forcedeth.c b/drivers/net/ethernet/nvidia/forcedeth.c |
| index 486cbc8ab224..7a549b834e97 100644 |
| --- a/drivers/net/ethernet/nvidia/forcedeth.c |
| +++ b/drivers/net/ethernet/nvidia/forcedeth.c |
| @@ -1734,12 +1734,12 @@ static void nv_get_stats(int cpu, struct fe_priv *np, |
| u64 tx_packets, tx_bytes, tx_dropped; |
| |
| do { |
| - syncp_start = u64_stats_fetch_begin_irq(&np->swstats_rx_syncp); |
| + syncp_start = u64_stats_fetch_begin(&np->swstats_rx_syncp); |
| rx_packets = src->stat_rx_packets; |
| rx_bytes = src->stat_rx_bytes; |
| rx_dropped = src->stat_rx_dropped; |
| rx_missed_errors = src->stat_rx_missed_errors; |
| - } while (u64_stats_fetch_retry_irq(&np->swstats_rx_syncp, syncp_start)); |
| + } while (u64_stats_fetch_retry(&np->swstats_rx_syncp, syncp_start)); |
| |
| storage->rx_packets += rx_packets; |
| storage->rx_bytes += rx_bytes; |
| @@ -1747,11 +1747,11 @@ static void nv_get_stats(int cpu, struct fe_priv *np, |
| storage->rx_missed_errors += rx_missed_errors; |
| |
| do { |
| - syncp_start = u64_stats_fetch_begin_irq(&np->swstats_tx_syncp); |
| + syncp_start = u64_stats_fetch_begin(&np->swstats_tx_syncp); |
| tx_packets = src->stat_tx_packets; |
| tx_bytes = src->stat_tx_bytes; |
| tx_dropped = src->stat_tx_dropped; |
| - } while (u64_stats_fetch_retry_irq(&np->swstats_tx_syncp, syncp_start)); |
| + } while (u64_stats_fetch_retry(&np->swstats_tx_syncp, syncp_start)); |
| |
| storage->tx_packets += tx_packets; |
| storage->tx_bytes += tx_bytes; |
| diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c |
| index 1b2119b1d48a..3f5e6572d20e 100644 |
| --- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c |
| +++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c |
| @@ -135,9 +135,9 @@ static void rmnet_get_stats64(struct net_device *dev, |
| pcpu_ptr = per_cpu_ptr(priv->pcpu_stats, cpu); |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&pcpu_ptr->syncp); |
| + start = u64_stats_fetch_begin(&pcpu_ptr->syncp); |
| snapshot = pcpu_ptr->stats; /* struct assignment */ |
| - } while (u64_stats_fetch_retry_irq(&pcpu_ptr->syncp, start)); |
| + } while (u64_stats_fetch_retry(&pcpu_ptr->syncp, start)); |
| |
| total_stats.rx_pkts += snapshot.rx_pkts; |
| total_stats.rx_bytes += snapshot.rx_bytes; |
| diff --git a/drivers/net/ethernet/realtek/8139too.c b/drivers/net/ethernet/realtek/8139too.c |
| index 469e2e229c6e..9ce0e8a64ba8 100644 |
| --- a/drivers/net/ethernet/realtek/8139too.c |
| +++ b/drivers/net/ethernet/realtek/8139too.c |
| @@ -2532,16 +2532,16 @@ rtl8139_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) |
| netdev_stats_to_stats64(stats, &dev->stats); |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&tp->rx_stats.syncp); |
| + start = u64_stats_fetch_begin(&tp->rx_stats.syncp); |
| stats->rx_packets = tp->rx_stats.packets; |
| stats->rx_bytes = tp->rx_stats.bytes; |
| - } while (u64_stats_fetch_retry_irq(&tp->rx_stats.syncp, start)); |
| + } while (u64_stats_fetch_retry(&tp->rx_stats.syncp, start)); |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&tp->tx_stats.syncp); |
| + start = u64_stats_fetch_begin(&tp->tx_stats.syncp); |
| stats->tx_packets = tp->tx_stats.packets; |
| stats->tx_bytes = tp->tx_stats.bytes; |
| - } while (u64_stats_fetch_retry_irq(&tp->tx_stats.syncp, start)); |
| + } while (u64_stats_fetch_retry(&tp->tx_stats.syncp, start)); |
| } |
| |
| /* Set or clear the multicast filter for this adaptor. |
| diff --git a/drivers/net/ethernet/socionext/sni_ave.c b/drivers/net/ethernet/socionext/sni_ave.c |
| index d2c6a5dfdc0e..b7e24ae92525 100644 |
| --- a/drivers/net/ethernet/socionext/sni_ave.c |
| +++ b/drivers/net/ethernet/socionext/sni_ave.c |
| @@ -1508,16 +1508,16 @@ static void ave_get_stats64(struct net_device *ndev, |
| unsigned int start; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&priv->stats_rx.syncp); |
| + start = u64_stats_fetch_begin(&priv->stats_rx.syncp); |
| stats->rx_packets = priv->stats_rx.packets; |
| stats->rx_bytes = priv->stats_rx.bytes; |
| - } while (u64_stats_fetch_retry_irq(&priv->stats_rx.syncp, start)); |
| + } while (u64_stats_fetch_retry(&priv->stats_rx.syncp, start)); |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&priv->stats_tx.syncp); |
| + start = u64_stats_fetch_begin(&priv->stats_tx.syncp); |
| stats->tx_packets = priv->stats_tx.packets; |
| stats->tx_bytes = priv->stats_tx.bytes; |
| - } while (u64_stats_fetch_retry_irq(&priv->stats_tx.syncp, start)); |
| + } while (u64_stats_fetch_retry(&priv->stats_tx.syncp, start)); |
| |
| stats->rx_errors = priv->stats_rx.errors; |
| stats->tx_errors = priv->stats_tx.errors; |
| diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.c b/drivers/net/ethernet/ti/am65-cpsw-nuss.c |
| index 76fabeae512d..3069032be5b9 100644 |
| --- a/drivers/net/ethernet/ti/am65-cpsw-nuss.c |
| +++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.c |
| @@ -1376,12 +1376,12 @@ static void am65_cpsw_nuss_ndo_get_stats(struct net_device *dev, |
| |
| cpu_stats = per_cpu_ptr(ndev_priv->stats, cpu); |
| do { |
| - start = u64_stats_fetch_begin_irq(&cpu_stats->syncp); |
| + start = u64_stats_fetch_begin(&cpu_stats->syncp); |
| rx_packets = cpu_stats->rx_packets; |
| rx_bytes = cpu_stats->rx_bytes; |
| tx_packets = cpu_stats->tx_packets; |
| tx_bytes = cpu_stats->tx_bytes; |
| - } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&cpu_stats->syncp, start)); |
| |
| stats->rx_packets += rx_packets; |
| stats->rx_bytes += rx_bytes; |
| diff --git a/drivers/net/ethernet/ti/netcp_core.c b/drivers/net/ethernet/ti/netcp_core.c |
| index 9eb9eaff4dc9..1bb596a9d8a2 100644 |
| --- a/drivers/net/ethernet/ti/netcp_core.c |
| +++ b/drivers/net/ethernet/ti/netcp_core.c |
| @@ -1916,16 +1916,16 @@ netcp_get_stats(struct net_device *ndev, struct rtnl_link_stats64 *stats) |
| unsigned int start; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&p->syncp_rx); |
| + start = u64_stats_fetch_begin(&p->syncp_rx); |
| rxpackets = p->rx_packets; |
| rxbytes = p->rx_bytes; |
| - } while (u64_stats_fetch_retry_irq(&p->syncp_rx, start)); |
| + } while (u64_stats_fetch_retry(&p->syncp_rx, start)); |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&p->syncp_tx); |
| + start = u64_stats_fetch_begin(&p->syncp_tx); |
| txpackets = p->tx_packets; |
| txbytes = p->tx_bytes; |
| - } while (u64_stats_fetch_retry_irq(&p->syncp_tx, start)); |
| + } while (u64_stats_fetch_retry(&p->syncp_tx, start)); |
| |
| stats->rx_packets = rxpackets; |
| stats->rx_bytes = rxbytes; |
| diff --git a/drivers/net/ethernet/via/via-rhine.c b/drivers/net/ethernet/via/via-rhine.c |
| index 0fb15a17b547..d716e6fe26e1 100644 |
| --- a/drivers/net/ethernet/via/via-rhine.c |
| +++ b/drivers/net/ethernet/via/via-rhine.c |
| @@ -2217,16 +2217,16 @@ rhine_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) |
| netdev_stats_to_stats64(stats, &dev->stats); |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&rp->rx_stats.syncp); |
| + start = u64_stats_fetch_begin(&rp->rx_stats.syncp); |
| stats->rx_packets = rp->rx_stats.packets; |
| stats->rx_bytes = rp->rx_stats.bytes; |
| - } while (u64_stats_fetch_retry_irq(&rp->rx_stats.syncp, start)); |
| + } while (u64_stats_fetch_retry(&rp->rx_stats.syncp, start)); |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&rp->tx_stats.syncp); |
| + start = u64_stats_fetch_begin(&rp->tx_stats.syncp); |
| stats->tx_packets = rp->tx_stats.packets; |
| stats->tx_bytes = rp->tx_stats.bytes; |
| - } while (u64_stats_fetch_retry_irq(&rp->tx_stats.syncp, start)); |
| + } while (u64_stats_fetch_retry(&rp->tx_stats.syncp, start)); |
| } |
| |
| static void rhine_set_rx_mode(struct net_device *dev) |
| diff --git a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c |
| index 5ea9dc251dd9..c678876a7826 100644 |
| --- a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c |
| +++ b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c |
| @@ -1305,16 +1305,16 @@ axienet_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) |
| netdev_stats_to_stats64(stats, &dev->stats); |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&lp->rx_stat_sync); |
| + start = u64_stats_fetch_begin(&lp->rx_stat_sync); |
| stats->rx_packets = u64_stats_read(&lp->rx_packets); |
| stats->rx_bytes = u64_stats_read(&lp->rx_bytes); |
| - } while (u64_stats_fetch_retry_irq(&lp->rx_stat_sync, start)); |
| + } while (u64_stats_fetch_retry(&lp->rx_stat_sync, start)); |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&lp->tx_stat_sync); |
| + start = u64_stats_fetch_begin(&lp->tx_stat_sync); |
| stats->tx_packets = u64_stats_read(&lp->tx_packets); |
| stats->tx_bytes = u64_stats_read(&lp->tx_bytes); |
| - } while (u64_stats_fetch_retry_irq(&lp->tx_stat_sync, start)); |
| + } while (u64_stats_fetch_retry(&lp->tx_stat_sync, start)); |
| } |
| |
| static const struct net_device_ops axienet_netdev_ops = { |
| diff --git a/drivers/net/hyperv/netvsc_drv.c b/drivers/net/hyperv/netvsc_drv.c |
| index 1b7405539984..453ff84b784a 100644 |
| --- a/drivers/net/hyperv/netvsc_drv.c |
| +++ b/drivers/net/hyperv/netvsc_drv.c |
| @@ -1268,12 +1268,12 @@ static void netvsc_get_vf_stats(struct net_device *net, |
| unsigned int start; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&stats->syncp); |
| + start = u64_stats_fetch_begin(&stats->syncp); |
| rx_packets = stats->rx_packets; |
| tx_packets = stats->tx_packets; |
| rx_bytes = stats->rx_bytes; |
| tx_bytes = stats->tx_bytes; |
| - } while (u64_stats_fetch_retry_irq(&stats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&stats->syncp, start)); |
| |
| tot->rx_packets += rx_packets; |
| tot->tx_packets += tx_packets; |
| @@ -1298,12 +1298,12 @@ static void netvsc_get_pcpu_stats(struct net_device *net, |
| unsigned int start; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&stats->syncp); |
| + start = u64_stats_fetch_begin(&stats->syncp); |
| this_tot->vf_rx_packets = stats->rx_packets; |
| this_tot->vf_tx_packets = stats->tx_packets; |
| this_tot->vf_rx_bytes = stats->rx_bytes; |
| this_tot->vf_tx_bytes = stats->tx_bytes; |
| - } while (u64_stats_fetch_retry_irq(&stats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&stats->syncp, start)); |
| this_tot->rx_packets = this_tot->vf_rx_packets; |
| this_tot->tx_packets = this_tot->vf_tx_packets; |
| this_tot->rx_bytes = this_tot->vf_rx_bytes; |
| @@ -1322,20 +1322,20 @@ static void netvsc_get_pcpu_stats(struct net_device *net, |
| |
| tx_stats = &nvchan->tx_stats; |
| do { |
| - start = u64_stats_fetch_begin_irq(&tx_stats->syncp); |
| + start = u64_stats_fetch_begin(&tx_stats->syncp); |
| packets = tx_stats->packets; |
| bytes = tx_stats->bytes; |
| - } while (u64_stats_fetch_retry_irq(&tx_stats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&tx_stats->syncp, start)); |
| |
| this_tot->tx_bytes += bytes; |
| this_tot->tx_packets += packets; |
| |
| rx_stats = &nvchan->rx_stats; |
| do { |
| - start = u64_stats_fetch_begin_irq(&rx_stats->syncp); |
| + start = u64_stats_fetch_begin(&rx_stats->syncp); |
| packets = rx_stats->packets; |
| bytes = rx_stats->bytes; |
| - } while (u64_stats_fetch_retry_irq(&rx_stats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&rx_stats->syncp, start)); |
| |
| this_tot->rx_bytes += bytes; |
| this_tot->rx_packets += packets; |
| @@ -1374,21 +1374,21 @@ static void netvsc_get_stats64(struct net_device *net, |
| |
| tx_stats = &nvchan->tx_stats; |
| do { |
| - start = u64_stats_fetch_begin_irq(&tx_stats->syncp); |
| + start = u64_stats_fetch_begin(&tx_stats->syncp); |
| packets = tx_stats->packets; |
| bytes = tx_stats->bytes; |
| - } while (u64_stats_fetch_retry_irq(&tx_stats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&tx_stats->syncp, start)); |
| |
| t->tx_bytes += bytes; |
| t->tx_packets += packets; |
| |
| rx_stats = &nvchan->rx_stats; |
| do { |
| - start = u64_stats_fetch_begin_irq(&rx_stats->syncp); |
| + start = u64_stats_fetch_begin(&rx_stats->syncp); |
| packets = rx_stats->packets; |
| bytes = rx_stats->bytes; |
| multicast = rx_stats->multicast + rx_stats->broadcast; |
| - } while (u64_stats_fetch_retry_irq(&rx_stats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&rx_stats->syncp, start)); |
| |
| t->rx_bytes += bytes; |
| t->rx_packets += packets; |
| @@ -1531,24 +1531,24 @@ static void netvsc_get_ethtool_stats(struct net_device *dev, |
| tx_stats = &nvdev->chan_table[j].tx_stats; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&tx_stats->syncp); |
| + start = u64_stats_fetch_begin(&tx_stats->syncp); |
| packets = tx_stats->packets; |
| bytes = tx_stats->bytes; |
| xdp_xmit = tx_stats->xdp_xmit; |
| - } while (u64_stats_fetch_retry_irq(&tx_stats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&tx_stats->syncp, start)); |
| data[i++] = packets; |
| data[i++] = bytes; |
| data[i++] = xdp_xmit; |
| |
| rx_stats = &nvdev->chan_table[j].rx_stats; |
| do { |
| - start = u64_stats_fetch_begin_irq(&rx_stats->syncp); |
| + start = u64_stats_fetch_begin(&rx_stats->syncp); |
| packets = rx_stats->packets; |
| bytes = rx_stats->bytes; |
| xdp_drop = rx_stats->xdp_drop; |
| xdp_redirect = rx_stats->xdp_redirect; |
| xdp_tx = rx_stats->xdp_tx; |
| - } while (u64_stats_fetch_retry_irq(&rx_stats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&rx_stats->syncp, start)); |
| data[i++] = packets; |
| data[i++] = bytes; |
| data[i++] = xdp_drop; |
| diff --git a/drivers/net/ifb.c b/drivers/net/ifb.c |
| index 1c64d5347b8e..78253ad57b2e 100644 |
| --- a/drivers/net/ifb.c |
| +++ b/drivers/net/ifb.c |
| @@ -162,18 +162,18 @@ static void ifb_stats64(struct net_device *dev, |
| |
| for (i = 0; i < dev->num_tx_queues; i++,txp++) { |
| do { |
| - start = u64_stats_fetch_begin_irq(&txp->rx_stats.sync); |
| + start = u64_stats_fetch_begin(&txp->rx_stats.sync); |
| packets = txp->rx_stats.packets; |
| bytes = txp->rx_stats.bytes; |
| - } while (u64_stats_fetch_retry_irq(&txp->rx_stats.sync, start)); |
| + } while (u64_stats_fetch_retry(&txp->rx_stats.sync, start)); |
| stats->rx_packets += packets; |
| stats->rx_bytes += bytes; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&txp->tx_stats.sync); |
| + start = u64_stats_fetch_begin(&txp->tx_stats.sync); |
| packets = txp->tx_stats.packets; |
| bytes = txp->tx_stats.bytes; |
| - } while (u64_stats_fetch_retry_irq(&txp->tx_stats.sync, start)); |
| + } while (u64_stats_fetch_retry(&txp->tx_stats.sync, start)); |
| stats->tx_packets += packets; |
| stats->tx_bytes += bytes; |
| } |
| @@ -245,12 +245,12 @@ static void ifb_fill_stats_data(u64 **data, |
| int j; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&q_stats->sync); |
| + start = u64_stats_fetch_begin(&q_stats->sync); |
| for (j = 0; j < IFB_Q_STATS_LEN; j++) { |
| offset = ifb_q_stats_desc[j].offset; |
| (*data)[j] = *(u64 *)(stats_base + offset); |
| } |
| - } while (u64_stats_fetch_retry_irq(&q_stats->sync, start)); |
| + } while (u64_stats_fetch_retry(&q_stats->sync, start)); |
| |
| *data += IFB_Q_STATS_LEN; |
| } |
| diff --git a/drivers/net/ipvlan/ipvlan_main.c b/drivers/net/ipvlan/ipvlan_main.c |
| index fbf2d5b67aaf..57c79f5f2991 100644 |
| --- a/drivers/net/ipvlan/ipvlan_main.c |
| +++ b/drivers/net/ipvlan/ipvlan_main.c |
| @@ -301,13 +301,13 @@ static void ipvlan_get_stats64(struct net_device *dev, |
| for_each_possible_cpu(idx) { |
| pcptr = per_cpu_ptr(ipvlan->pcpu_stats, idx); |
| do { |
| - strt= u64_stats_fetch_begin_irq(&pcptr->syncp); |
| + strt = u64_stats_fetch_begin(&pcptr->syncp); |
| rx_pkts = u64_stats_read(&pcptr->rx_pkts); |
| rx_bytes = u64_stats_read(&pcptr->rx_bytes); |
| rx_mcast = u64_stats_read(&pcptr->rx_mcast); |
| tx_pkts = u64_stats_read(&pcptr->tx_pkts); |
| tx_bytes = u64_stats_read(&pcptr->tx_bytes); |
| - } while (u64_stats_fetch_retry_irq(&pcptr->syncp, |
| + } while (u64_stats_fetch_retry(&pcptr->syncp, |
| strt)); |
| |
| s->rx_packets += rx_pkts; |
| diff --git a/drivers/net/loopback.c b/drivers/net/loopback.c |
| index 2e9742952c4e..f6d53e63ef4e 100644 |
| --- a/drivers/net/loopback.c |
| +++ b/drivers/net/loopback.c |
| @@ -106,10 +106,10 @@ void dev_lstats_read(struct net_device *dev, u64 *packets, u64 *bytes) |
| |
| lb_stats = per_cpu_ptr(dev->lstats, i); |
| do { |
| - start = u64_stats_fetch_begin_irq(&lb_stats->syncp); |
| + start = u64_stats_fetch_begin(&lb_stats->syncp); |
| tpackets = u64_stats_read(&lb_stats->packets); |
| tbytes = u64_stats_read(&lb_stats->bytes); |
| - } while (u64_stats_fetch_retry_irq(&lb_stats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&lb_stats->syncp, start)); |
| *bytes += tbytes; |
| *packets += tpackets; |
| } |
| diff --git a/drivers/net/macsec.c b/drivers/net/macsec.c |
| index 209ee9f35275..f9f25b5f1745 100644 |
| --- a/drivers/net/macsec.c |
| +++ b/drivers/net/macsec.c |
| @@ -2802,9 +2802,9 @@ static void get_rx_sc_stats(struct net_device *dev, |
| |
| stats = per_cpu_ptr(rx_sc->stats, cpu); |
| do { |
| - start = u64_stats_fetch_begin_irq(&stats->syncp); |
| + start = u64_stats_fetch_begin(&stats->syncp); |
| memcpy(&tmp, &stats->stats, sizeof(tmp)); |
| - } while (u64_stats_fetch_retry_irq(&stats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&stats->syncp, start)); |
| |
| sum->InOctetsValidated += tmp.InOctetsValidated; |
| sum->InOctetsDecrypted += tmp.InOctetsDecrypted; |
| @@ -2883,9 +2883,9 @@ static void get_tx_sc_stats(struct net_device *dev, |
| |
| stats = per_cpu_ptr(macsec_priv(dev)->secy.tx_sc.stats, cpu); |
| do { |
| - start = u64_stats_fetch_begin_irq(&stats->syncp); |
| + start = u64_stats_fetch_begin(&stats->syncp); |
| memcpy(&tmp, &stats->stats, sizeof(tmp)); |
| - } while (u64_stats_fetch_retry_irq(&stats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&stats->syncp, start)); |
| |
| sum->OutPktsProtected += tmp.OutPktsProtected; |
| sum->OutPktsEncrypted += tmp.OutPktsEncrypted; |
| @@ -2939,9 +2939,9 @@ static void get_secy_stats(struct net_device *dev, struct macsec_dev_stats *sum) |
| |
| stats = per_cpu_ptr(macsec_priv(dev)->stats, cpu); |
| do { |
| - start = u64_stats_fetch_begin_irq(&stats->syncp); |
| + start = u64_stats_fetch_begin(&stats->syncp); |
| memcpy(&tmp, &stats->stats, sizeof(tmp)); |
| - } while (u64_stats_fetch_retry_irq(&stats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&stats->syncp, start)); |
| |
| sum->OutPktsUntagged += tmp.OutPktsUntagged; |
| sum->InPktsUntagged += tmp.InPktsUntagged; |
| diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c |
| index 012830d12fde..9bea3f1b0a8a 100644 |
| --- a/drivers/net/macvlan.c |
| +++ b/drivers/net/macvlan.c |
| @@ -948,13 +948,13 @@ static void macvlan_dev_get_stats64(struct net_device *dev, |
| for_each_possible_cpu(i) { |
| p = per_cpu_ptr(vlan->pcpu_stats, i); |
| do { |
| - start = u64_stats_fetch_begin_irq(&p->syncp); |
| + start = u64_stats_fetch_begin(&p->syncp); |
| rx_packets = u64_stats_read(&p->rx_packets); |
| rx_bytes = u64_stats_read(&p->rx_bytes); |
| rx_multicast = u64_stats_read(&p->rx_multicast); |
| tx_packets = u64_stats_read(&p->tx_packets); |
| tx_bytes = u64_stats_read(&p->tx_bytes); |
| - } while (u64_stats_fetch_retry_irq(&p->syncp, start)); |
| + } while (u64_stats_fetch_retry(&p->syncp, start)); |
| |
| stats->rx_packets += rx_packets; |
| stats->rx_bytes += rx_bytes; |
| diff --git a/drivers/net/mhi_net.c b/drivers/net/mhi_net.c |
| index 0b9d37979133..3d322ac4f6a5 100644 |
| --- a/drivers/net/mhi_net.c |
| +++ b/drivers/net/mhi_net.c |
| @@ -104,19 +104,19 @@ static void mhi_ndo_get_stats64(struct net_device *ndev, |
| unsigned int start; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&mhi_netdev->stats.rx_syncp); |
| + start = u64_stats_fetch_begin(&mhi_netdev->stats.rx_syncp); |
| stats->rx_packets = u64_stats_read(&mhi_netdev->stats.rx_packets); |
| stats->rx_bytes = u64_stats_read(&mhi_netdev->stats.rx_bytes); |
| stats->rx_errors = u64_stats_read(&mhi_netdev->stats.rx_errors); |
| - } while (u64_stats_fetch_retry_irq(&mhi_netdev->stats.rx_syncp, start)); |
| + } while (u64_stats_fetch_retry(&mhi_netdev->stats.rx_syncp, start)); |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&mhi_netdev->stats.tx_syncp); |
| + start = u64_stats_fetch_begin(&mhi_netdev->stats.tx_syncp); |
| stats->tx_packets = u64_stats_read(&mhi_netdev->stats.tx_packets); |
| stats->tx_bytes = u64_stats_read(&mhi_netdev->stats.tx_bytes); |
| stats->tx_errors = u64_stats_read(&mhi_netdev->stats.tx_errors); |
| stats->tx_dropped = u64_stats_read(&mhi_netdev->stats.tx_dropped); |
| - } while (u64_stats_fetch_retry_irq(&mhi_netdev->stats.tx_syncp, start)); |
| + } while (u64_stats_fetch_retry(&mhi_netdev->stats.tx_syncp, start)); |
| } |
| |
| static const struct net_device_ops mhi_netdev_ops = { |
| diff --git a/drivers/net/netdevsim/netdev.c b/drivers/net/netdevsim/netdev.c |
| index 9a1a5b203624..e470e3398abc 100644 |
| --- a/drivers/net/netdevsim/netdev.c |
| +++ b/drivers/net/netdevsim/netdev.c |
| @@ -67,10 +67,10 @@ nsim_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) |
| unsigned int start; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&ns->syncp); |
| + start = u64_stats_fetch_begin(&ns->syncp); |
| stats->tx_bytes = ns->tx_bytes; |
| stats->tx_packets = ns->tx_packets; |
| - } while (u64_stats_fetch_retry_irq(&ns->syncp, start)); |
| + } while (u64_stats_fetch_retry(&ns->syncp, start)); |
| } |
| |
| static int |
| diff --git a/drivers/net/team/team.c b/drivers/net/team/team.c |
| index 293eaf6b3ec9..eccf9df0c88c 100644 |
| --- a/drivers/net/team/team.c |
| +++ b/drivers/net/team/team.c |
| @@ -1868,13 +1868,13 @@ team_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) |
| for_each_possible_cpu(i) { |
| p = per_cpu_ptr(team->pcpu_stats, i); |
| do { |
| - start = u64_stats_fetch_begin_irq(&p->syncp); |
| + start = u64_stats_fetch_begin(&p->syncp); |
| rx_packets = u64_stats_read(&p->rx_packets); |
| rx_bytes = u64_stats_read(&p->rx_bytes); |
| rx_multicast = u64_stats_read(&p->rx_multicast); |
| tx_packets = u64_stats_read(&p->tx_packets); |
| tx_bytes = u64_stats_read(&p->tx_bytes); |
| - } while (u64_stats_fetch_retry_irq(&p->syncp, start)); |
| + } while (u64_stats_fetch_retry(&p->syncp, start)); |
| |
| stats->rx_packets += rx_packets; |
| stats->rx_bytes += rx_bytes; |
| diff --git a/drivers/net/team/team_mode_loadbalance.c b/drivers/net/team/team_mode_loadbalance.c |
| index b095a4b4957b..18d99fda997c 100644 |
| --- a/drivers/net/team/team_mode_loadbalance.c |
| +++ b/drivers/net/team/team_mode_loadbalance.c |
| @@ -466,9 +466,9 @@ static void __lb_one_cpu_stats_add(struct lb_stats *acc_stats, |
| struct lb_stats tmp; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(syncp); |
| + start = u64_stats_fetch_begin(syncp); |
| tmp.tx_bytes = cpu_stats->tx_bytes; |
| - } while (u64_stats_fetch_retry_irq(syncp, start)); |
| + } while (u64_stats_fetch_retry(syncp, start)); |
| acc_stats->tx_bytes += tmp.tx_bytes; |
| } |
| |
| diff --git a/drivers/net/veth.c b/drivers/net/veth.c |
| index 8dcd3b6e143b..cfc81e45d461 100644 |
| --- a/drivers/net/veth.c |
| +++ b/drivers/net/veth.c |
| @@ -182,12 +182,12 @@ static void veth_get_ethtool_stats(struct net_device *dev, |
| size_t offset; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&rq_stats->syncp); |
| + start = u64_stats_fetch_begin(&rq_stats->syncp); |
| for (j = 0; j < VETH_RQ_STATS_LEN; j++) { |
| offset = veth_rq_stats_desc[j].offset; |
| data[idx + j] = *(u64 *)(stats_base + offset); |
| } |
| - } while (u64_stats_fetch_retry_irq(&rq_stats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&rq_stats->syncp, start)); |
| idx += VETH_RQ_STATS_LEN; |
| } |
| |
| @@ -203,12 +203,12 @@ static void veth_get_ethtool_stats(struct net_device *dev, |
| |
| tx_idx += (i % dev->real_num_tx_queues) * VETH_TQ_STATS_LEN; |
| do { |
| - start = u64_stats_fetch_begin_irq(&rq_stats->syncp); |
| + start = u64_stats_fetch_begin(&rq_stats->syncp); |
| for (j = 0; j < VETH_TQ_STATS_LEN; j++) { |
| offset = veth_tq_stats_desc[j].offset; |
| data[tx_idx + j] += *(u64 *)(base + offset); |
| } |
| - } while (u64_stats_fetch_retry_irq(&rq_stats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&rq_stats->syncp, start)); |
| } |
| } |
| |
| @@ -381,13 +381,13 @@ static void veth_stats_rx(struct veth_stats *result, struct net_device *dev) |
| unsigned int start; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&stats->syncp); |
| + start = u64_stats_fetch_begin(&stats->syncp); |
| peer_tq_xdp_xmit_err = stats->vs.peer_tq_xdp_xmit_err; |
| xdp_tx_err = stats->vs.xdp_tx_err; |
| packets = stats->vs.xdp_packets; |
| bytes = stats->vs.xdp_bytes; |
| drops = stats->vs.rx_drops; |
| - } while (u64_stats_fetch_retry_irq(&stats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&stats->syncp, start)); |
| result->peer_tq_xdp_xmit_err += peer_tq_xdp_xmit_err; |
| result->xdp_tx_err += xdp_tx_err; |
| result->xdp_packets += packets; |
| diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c |
| index 45f1a871b7da..59a2299f00d9 100644 |
| --- a/drivers/net/virtio_net.c |
| +++ b/drivers/net/virtio_net.c |
| @@ -2107,18 +2107,18 @@ static void virtnet_stats(struct net_device *dev, |
| struct send_queue *sq = &vi->sq[i]; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&sq->stats.syncp); |
| + start = u64_stats_fetch_begin(&sq->stats.syncp); |
| tpackets = sq->stats.packets; |
| tbytes = sq->stats.bytes; |
| terrors = sq->stats.tx_timeouts; |
| - } while (u64_stats_fetch_retry_irq(&sq->stats.syncp, start)); |
| + } while (u64_stats_fetch_retry(&sq->stats.syncp, start)); |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&rq->stats.syncp); |
| + start = u64_stats_fetch_begin(&rq->stats.syncp); |
| rpackets = rq->stats.packets; |
| rbytes = rq->stats.bytes; |
| rdrops = rq->stats.drops; |
| - } while (u64_stats_fetch_retry_irq(&rq->stats.syncp, start)); |
| + } while (u64_stats_fetch_retry(&rq->stats.syncp, start)); |
| |
| tot->rx_packets += rpackets; |
| tot->tx_packets += tpackets; |
| @@ -2726,12 +2726,12 @@ static void virtnet_get_ethtool_stats(struct net_device *dev, |
| |
| stats_base = (u8 *)&rq->stats; |
| do { |
| - start = u64_stats_fetch_begin_irq(&rq->stats.syncp); |
| + start = u64_stats_fetch_begin(&rq->stats.syncp); |
| for (j = 0; j < VIRTNET_RQ_STATS_LEN; j++) { |
| offset = virtnet_rq_stats_desc[j].offset; |
| data[idx + j] = *(u64 *)(stats_base + offset); |
| } |
| - } while (u64_stats_fetch_retry_irq(&rq->stats.syncp, start)); |
| + } while (u64_stats_fetch_retry(&rq->stats.syncp, start)); |
| idx += VIRTNET_RQ_STATS_LEN; |
| } |
| |
| @@ -2740,12 +2740,12 @@ static void virtnet_get_ethtool_stats(struct net_device *dev, |
| |
| stats_base = (u8 *)&sq->stats; |
| do { |
| - start = u64_stats_fetch_begin_irq(&sq->stats.syncp); |
| + start = u64_stats_fetch_begin(&sq->stats.syncp); |
| for (j = 0; j < VIRTNET_SQ_STATS_LEN; j++) { |
| offset = virtnet_sq_stats_desc[j].offset; |
| data[idx + j] = *(u64 *)(stats_base + offset); |
| } |
| - } while (u64_stats_fetch_retry_irq(&sq->stats.syncp, start)); |
| + } while (u64_stats_fetch_retry(&sq->stats.syncp, start)); |
| idx += VIRTNET_SQ_STATS_LEN; |
| } |
| } |
| diff --git a/drivers/net/vrf.c b/drivers/net/vrf.c |
| index 208df4d41939..6043e63b42f9 100644 |
| --- a/drivers/net/vrf.c |
| +++ b/drivers/net/vrf.c |
| @@ -159,13 +159,13 @@ static void vrf_get_stats64(struct net_device *dev, |
| |
| dstats = per_cpu_ptr(dev->dstats, i); |
| do { |
| - start = u64_stats_fetch_begin_irq(&dstats->syncp); |
| + start = u64_stats_fetch_begin(&dstats->syncp); |
| tbytes = dstats->tx_bytes; |
| tpkts = dstats->tx_pkts; |
| tdrops = dstats->tx_drps; |
| rbytes = dstats->rx_bytes; |
| rpkts = dstats->rx_pkts; |
| - } while (u64_stats_fetch_retry_irq(&dstats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&dstats->syncp, start)); |
| stats->tx_bytes += tbytes; |
| stats->tx_packets += tpkts; |
| stats->tx_dropped += tdrops; |
| diff --git a/drivers/net/vxlan/vxlan_vnifilter.c b/drivers/net/vxlan/vxlan_vnifilter.c |
| index c5cf55030158..c3ff30ab782e 100644 |
| --- a/drivers/net/vxlan/vxlan_vnifilter.c |
| +++ b/drivers/net/vxlan/vxlan_vnifilter.c |
| @@ -129,9 +129,9 @@ static void vxlan_vnifilter_stats_get(const struct vxlan_vni_node *vninode, |
| |
| pstats = per_cpu_ptr(vninode->stats, i); |
| do { |
| - start = u64_stats_fetch_begin_irq(&pstats->syncp); |
| + start = u64_stats_fetch_begin(&pstats->syncp); |
| memcpy(&temp, &pstats->stats, sizeof(temp)); |
| - } while (u64_stats_fetch_retry_irq(&pstats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&pstats->syncp, start)); |
| |
| dest->rx_packets += temp.rx_packets; |
| dest->rx_bytes += temp.rx_bytes; |
| diff --git a/drivers/net/wwan/mhi_wwan_mbim.c b/drivers/net/wwan/mhi_wwan_mbim.c |
| index ef70bb7c88ad..3f72ae943b29 100644 |
| --- a/drivers/net/wwan/mhi_wwan_mbim.c |
| +++ b/drivers/net/wwan/mhi_wwan_mbim.c |
| @@ -456,19 +456,19 @@ static void mhi_mbim_ndo_get_stats64(struct net_device *ndev, |
| unsigned int start; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&link->rx_syncp); |
| + start = u64_stats_fetch_begin(&link->rx_syncp); |
| stats->rx_packets = u64_stats_read(&link->rx_packets); |
| stats->rx_bytes = u64_stats_read(&link->rx_bytes); |
| stats->rx_errors = u64_stats_read(&link->rx_errors); |
| - } while (u64_stats_fetch_retry_irq(&link->rx_syncp, start)); |
| + } while (u64_stats_fetch_retry(&link->rx_syncp, start)); |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&link->tx_syncp); |
| + start = u64_stats_fetch_begin(&link->tx_syncp); |
| stats->tx_packets = u64_stats_read(&link->tx_packets); |
| stats->tx_bytes = u64_stats_read(&link->tx_bytes); |
| stats->tx_errors = u64_stats_read(&link->tx_errors); |
| stats->tx_dropped = u64_stats_read(&link->tx_dropped); |
| - } while (u64_stats_fetch_retry_irq(&link->tx_syncp, start)); |
| + } while (u64_stats_fetch_retry(&link->tx_syncp, start)); |
| } |
| |
| static void mhi_mbim_ul_callback(struct mhi_device *mhi_dev, |
| diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c |
| index dc404e05970c..14aec417fa06 100644 |
| --- a/drivers/net/xen-netfront.c |
| +++ b/drivers/net/xen-netfront.c |
| @@ -1392,16 +1392,16 @@ static void xennet_get_stats64(struct net_device *dev, |
| unsigned int start; |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&tx_stats->syncp); |
| + start = u64_stats_fetch_begin(&tx_stats->syncp); |
| tx_packets = tx_stats->packets; |
| tx_bytes = tx_stats->bytes; |
| - } while (u64_stats_fetch_retry_irq(&tx_stats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&tx_stats->syncp, start)); |
| |
| do { |
| - start = u64_stats_fetch_begin_irq(&rx_stats->syncp); |
| + start = u64_stats_fetch_begin(&rx_stats->syncp); |
| rx_packets = rx_stats->packets; |
| rx_bytes = rx_stats->bytes; |
| - } while (u64_stats_fetch_retry_irq(&rx_stats->syncp, start)); |
| + } while (u64_stats_fetch_retry(&rx_stats->syncp, start)); |
| |
| tot->rx_packets += rx_packets; |
| tot->tx_packets += tx_packets; |
| diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c |
| index 19688f333e0b..f978e5eb7e7e 100644 |
| --- a/drivers/spi/spi.c |
| +++ b/drivers/spi/spi.c |
| @@ -127,10 +127,10 @@ do { \ |
| unsigned int start; \ |
| pcpu_stats = per_cpu_ptr(in, i); \ |
| do { \ |
| - start = u64_stats_fetch_begin_irq( \ |
| + start = u64_stats_fetch_begin( \ |
| &pcpu_stats->syncp); \ |
| inc = u64_stats_read(&pcpu_stats->field); \ |
| - } while (u64_stats_fetch_retry_irq( \ |
| + } while (u64_stats_fetch_retry( \ |
| &pcpu_stats->syncp, start)); \ |
| ret += inc; \ |
| } \ |
| diff --git a/drivers/tty/serial/8250/8250.h b/drivers/tty/serial/8250/8250.h |
| index eeb7b43ebe53..b17715d340c3 100644 |
| --- a/drivers/tty/serial/8250/8250.h |
| +++ b/drivers/tty/serial/8250/8250.h |
| @@ -176,12 +176,49 @@ static inline void serial_dl_write(struct uart_8250_port *up, int value) |
| up->dl_write(up, value); |
| } |
| |
| +static inline int serial8250_in_IER(struct uart_8250_port *up) |
| +{ |
| + struct uart_port *port = &up->port; |
| + unsigned long flags; |
| + bool is_console; |
| + int ier; |
| + |
| + is_console = uart_console(port); |
| + |
| + if (is_console) |
| + printk_cpu_sync_get_irqsave(flags); |
| + |
| + ier = serial_in(up, UART_IER); |
| + |
| + if (is_console) |
| + printk_cpu_sync_put_irqrestore(flags); |
| + |
| + return ier; |
| +} |
| + |
| +static inline void serial8250_set_IER(struct uart_8250_port *up, int ier) |
| +{ |
| + struct uart_port *port = &up->port; |
| + unsigned long flags; |
| + bool is_console; |
| + |
| + is_console = uart_console(port); |
| + |
| + if (is_console) |
| + printk_cpu_sync_get_irqsave(flags); |
| + |
| + serial_out(up, UART_IER, ier); |
| + |
| + if (is_console) |
| + printk_cpu_sync_put_irqrestore(flags); |
| +} |
| + |
| static inline bool serial8250_set_THRI(struct uart_8250_port *up) |
| { |
| if (up->ier & UART_IER_THRI) |
| return false; |
| up->ier |= UART_IER_THRI; |
| - serial_out(up, UART_IER, up->ier); |
| + serial8250_set_IER(up, up->ier); |
| return true; |
| } |
| |
| @@ -190,7 +227,7 @@ static inline bool serial8250_clear_THRI(struct uart_8250_port *up) |
| if (!(up->ier & UART_IER_THRI)) |
| return false; |
| up->ier &= ~UART_IER_THRI; |
| - serial_out(up, UART_IER, up->ier); |
| + serial8250_set_IER(up, up->ier); |
| return true; |
| } |
| |
| diff --git a/drivers/tty/serial/8250/8250_aspeed_vuart.c b/drivers/tty/serial/8250/8250_aspeed_vuart.c |
| index 9d2a7856784f..7cc6b527c088 100644 |
| --- a/drivers/tty/serial/8250/8250_aspeed_vuart.c |
| +++ b/drivers/tty/serial/8250/8250_aspeed_vuart.c |
| @@ -278,7 +278,7 @@ static void __aspeed_vuart_set_throttle(struct uart_8250_port *up, |
| up->ier &= ~irqs; |
| if (!throttle) |
| up->ier |= irqs; |
| - serial_out(up, UART_IER, up->ier); |
| + serial8250_set_IER(up, up->ier); |
| } |
| static void aspeed_vuart_set_throttle(struct uart_port *port, bool throttle) |
| { |
| diff --git a/drivers/tty/serial/8250/8250_bcm7271.c b/drivers/tty/serial/8250/8250_bcm7271.c |
| index ffc7f67e27e3..8b211e668bc0 100644 |
| --- a/drivers/tty/serial/8250/8250_bcm7271.c |
| +++ b/drivers/tty/serial/8250/8250_bcm7271.c |
| @@ -609,7 +609,7 @@ static int brcmuart_startup(struct uart_port *port) |
| * will handle this. |
| */ |
| up->ier &= ~UART_IER_RDI; |
| - serial_port_out(port, UART_IER, up->ier); |
| + serial8250_set_IER(up, up->ier); |
| |
| priv->tx_running = false; |
| priv->dma.rx_dma = NULL; |
| @@ -775,10 +775,12 @@ static int brcmuart_handle_irq(struct uart_port *p) |
| unsigned int iir = serial_port_in(p, UART_IIR); |
| struct brcmuart_priv *priv = p->private_data; |
| struct uart_8250_port *up = up_to_u8250p(p); |
| + unsigned long cs_flags; |
| unsigned int status; |
| unsigned long flags; |
| unsigned int ier; |
| unsigned int mcr; |
| + bool is_console; |
| int handled = 0; |
| |
| /* |
| @@ -789,6 +791,10 @@ static int brcmuart_handle_irq(struct uart_port *p) |
| spin_lock_irqsave(&p->lock, flags); |
| status = serial_port_in(p, UART_LSR); |
| if ((status & UART_LSR_DR) == 0) { |
| + is_console = uart_console(p); |
| + |
| + if (is_console) |
| + printk_cpu_sync_get_irqsave(cs_flags); |
| |
| ier = serial_port_in(p, UART_IER); |
| /* |
| @@ -809,6 +815,9 @@ static int brcmuart_handle_irq(struct uart_port *p) |
| serial_port_in(p, UART_RX); |
| } |
| |
| + if (is_console) |
| + printk_cpu_sync_put_irqrestore(cs_flags); |
| + |
| handled = 1; |
| } |
| spin_unlock_irqrestore(&p->lock, flags); |
| @@ -823,8 +832,10 @@ static enum hrtimer_restart brcmuart_hrtimer_func(struct hrtimer *t) |
| struct brcmuart_priv *priv = container_of(t, struct brcmuart_priv, hrt); |
| struct uart_port *p = priv->up; |
| struct uart_8250_port *up = up_to_u8250p(p); |
| + unsigned long cs_flags; |
| unsigned int status; |
| unsigned long flags; |
| + bool is_console; |
| |
| if (priv->shutdown) |
| return HRTIMER_NORESTART; |
| @@ -846,12 +857,20 @@ static enum hrtimer_restart brcmuart_hrtimer_func(struct hrtimer *t) |
| /* re-enable receive unless upper layer has disabled it */ |
| if ((up->ier & (UART_IER_RLSI | UART_IER_RDI)) == |
| (UART_IER_RLSI | UART_IER_RDI)) { |
| + is_console = uart_console(p); |
| + |
| + if (is_console) |
| + printk_cpu_sync_get_irqsave(cs_flags); |
| + |
| status = serial_port_in(p, UART_IER); |
| status |= (UART_IER_RLSI | UART_IER_RDI); |
| serial_port_out(p, UART_IER, status); |
| status = serial_port_in(p, UART_MCR); |
| status |= UART_MCR_RTS; |
| serial_port_out(p, UART_MCR, status); |
| + |
| + if (is_console) |
| + printk_cpu_sync_put_irqrestore(cs_flags); |
| } |
| spin_unlock_irqrestore(&p->lock, flags); |
| return HRTIMER_NORESTART; |
| diff --git a/drivers/tty/serial/8250/8250_core.c b/drivers/tty/serial/8250/8250_core.c |
| index 81a5dab1a828..536f639ff56c 100644 |
| --- a/drivers/tty/serial/8250/8250_core.c |
| +++ b/drivers/tty/serial/8250/8250_core.c |
| @@ -255,8 +255,11 @@ static void serial8250_timeout(struct timer_list *t) |
| static void serial8250_backup_timeout(struct timer_list *t) |
| { |
| struct uart_8250_port *up = from_timer(up, t, timer); |
| + struct uart_port *port = &up->port; |
| unsigned int iir, ier = 0, lsr; |
| + unsigned long cs_flags; |
| unsigned long flags; |
| + bool is_console; |
| |
| spin_lock_irqsave(&up->port.lock, flags); |
| |
| @@ -265,8 +268,16 @@ static void serial8250_backup_timeout(struct timer_list *t) |
| * based handler. |
| */ |
| if (up->port.irq) { |
| + is_console = uart_console(port); |
| + |
| + if (is_console) |
| + printk_cpu_sync_get_irqsave(cs_flags); |
| + |
| ier = serial_in(up, UART_IER); |
| serial_out(up, UART_IER, 0); |
| + |
| + if (is_console) |
| + printk_cpu_sync_put_irqrestore(cs_flags); |
| } |
| |
| iir = serial_in(up, UART_IIR); |
| @@ -289,7 +300,7 @@ static void serial8250_backup_timeout(struct timer_list *t) |
| serial8250_tx_chars(up); |
| |
| if (up->port.irq) |
| - serial_out(up, UART_IER, ier); |
| + serial8250_set_IER(up, ier); |
| |
| spin_unlock_irqrestore(&up->port.lock, flags); |
| |
| @@ -575,6 +586,14 @@ serial8250_register_ports(struct uart_driver *drv, struct device *dev) |
| |
| #ifdef CONFIG_SERIAL_8250_CONSOLE |
| |
| +static void univ8250_console_write_atomic(struct console *co, const char *s, |
| + unsigned int count) |
| +{ |
| + struct uart_8250_port *up = &serial8250_ports[co->index]; |
| + |
| + serial8250_console_write_atomic(up, s, count); |
| +} |
| + |
| static void univ8250_console_write(struct console *co, const char *s, |
| unsigned int count) |
| { |
| @@ -668,6 +687,7 @@ static int univ8250_console_match(struct console *co, char *name, int idx, |
| |
| static struct console univ8250_console = { |
| .name = "ttyS", |
| + .write_atomic = univ8250_console_write_atomic, |
| .write = univ8250_console_write, |
| .device = uart_console_device, |
| .setup = univ8250_console_setup, |
| @@ -961,7 +981,7 @@ static void serial_8250_overrun_backoff_work(struct work_struct *work) |
| spin_lock_irqsave(&port->lock, flags); |
| up->ier |= UART_IER_RLSI | UART_IER_RDI; |
| up->port.read_status_mask |= UART_LSR_DR; |
| - serial_out(up, UART_IER, up->ier); |
| + serial8250_set_IER(up, up->ier); |
| spin_unlock_irqrestore(&port->lock, flags); |
| } |
| |
| diff --git a/drivers/tty/serial/8250/8250_exar.c b/drivers/tty/serial/8250/8250_exar.c |
| index 55451ff84652..39f5c5933381 100644 |
| --- a/drivers/tty/serial/8250/8250_exar.c |
| +++ b/drivers/tty/serial/8250/8250_exar.c |
| @@ -189,6 +189,8 @@ static void xr17v35x_set_divisor(struct uart_port *p, unsigned int baud, |
| |
| static int xr17v35x_startup(struct uart_port *port) |
| { |
| + struct uart_8250_port *up = up_to_u8250p(port); |
| + |
| /* |
| * First enable access to IER [7:5], ISR [5:4], FCR [5:4], |
| * MCR [7:5] and MSR [7:0] |
| @@ -199,7 +201,7 @@ static int xr17v35x_startup(struct uart_port *port) |
| * Make sure all interrups are masked until initialization is |
| * complete and the FIFOs are cleared |
| */ |
| - serial_port_out(port, UART_IER, 0); |
| + serial8250_set_IER(up, 0); |
| |
| return serial8250_do_startup(port); |
| } |
| diff --git a/drivers/tty/serial/8250/8250_fsl.c b/drivers/tty/serial/8250/8250_fsl.c |
| index 8adfaa183f77..eaf148245a10 100644 |
| --- a/drivers/tty/serial/8250/8250_fsl.c |
| +++ b/drivers/tty/serial/8250/8250_fsl.c |
| @@ -58,7 +58,8 @@ int fsl8250_handle_irq(struct uart_port *port) |
| if ((orig_lsr & UART_LSR_OE) && (up->overrun_backoff_time_ms > 0)) { |
| unsigned long delay; |
| |
| - up->ier = port->serial_in(port, UART_IER); |
| + up->ier = serial8250_in_IER(up); |
| + |
| if (up->ier & (UART_IER_RLSI | UART_IER_RDI)) { |
| port->ops->stop_rx(port); |
| } else { |
| diff --git a/drivers/tty/serial/8250/8250_ingenic.c b/drivers/tty/serial/8250/8250_ingenic.c |
| index 2b2f5d8d24b9..2b78e6c394fb 100644 |
| --- a/drivers/tty/serial/8250/8250_ingenic.c |
| +++ b/drivers/tty/serial/8250/8250_ingenic.c |
| @@ -146,6 +146,7 @@ OF_EARLYCON_DECLARE(x1000_uart, "ingenic,x1000-uart", |
| |
| static void ingenic_uart_serial_out(struct uart_port *p, int offset, int value) |
| { |
| + struct uart_8250_port *up = up_to_u8250p(p); |
| int ier; |
| |
| switch (offset) { |
| @@ -167,7 +168,7 @@ static void ingenic_uart_serial_out(struct uart_port *p, int offset, int value) |
| * If we have enabled modem status IRQs we should enable |
| * modem mode. |
| */ |
| - ier = p->serial_in(p, UART_IER); |
| + ier = serial8250_in_IER(up); |
| |
| if (ier & UART_IER_MSI) |
| value |= UART_MCR_MDCE | UART_MCR_FCM; |
| diff --git a/drivers/tty/serial/8250/8250_mtk.c b/drivers/tty/serial/8250/8250_mtk.c |
| index fb1d5ec0940e..3e7203909d6a 100644 |
| --- a/drivers/tty/serial/8250/8250_mtk.c |
| +++ b/drivers/tty/serial/8250/8250_mtk.c |
| @@ -222,12 +222,40 @@ static void mtk8250_shutdown(struct uart_port *port) |
| |
| static void mtk8250_disable_intrs(struct uart_8250_port *up, int mask) |
| { |
| - serial_out(up, UART_IER, serial_in(up, UART_IER) & (~mask)); |
| + struct uart_port *port = &up->port; |
| + unsigned long flags; |
| + bool is_console; |
| + int ier; |
| + |
| + is_console = uart_console(port); |
| + |
| + if (is_console) |
| + printk_cpu_sync_get_irqsave(flags); |
| + |
| + ier = serial_in(up, UART_IER); |
| + serial_out(up, UART_IER, ier & (~mask)); |
| + |
| + if (is_console) |
| + printk_cpu_sync_put_irqrestore(flags); |
| } |
| |
| static void mtk8250_enable_intrs(struct uart_8250_port *up, int mask) |
| { |
| - serial_out(up, UART_IER, serial_in(up, UART_IER) | mask); |
| + struct uart_port *port = &up->port; |
| + unsigned long flags; |
| + bool is_console; |
| + int ier; |
| + |
| + is_console = uart_console(port); |
| + |
| + if (is_console) |
| + printk_cpu_sync_get_irqsave(flags); |
| + |
| + ier = serial_in(up, UART_IER); |
| + serial_out(up, UART_IER, ier | mask); |
| + |
| + if (is_console) |
| + printk_cpu_sync_put_irqrestore(flags); |
| } |
| |
| static void mtk8250_set_flow_ctrl(struct uart_8250_port *up, int mode) |
| diff --git a/drivers/tty/serial/8250/8250_omap.c b/drivers/tty/serial/8250/8250_omap.c |
| index 037d613006f5..4346adb6c3b1 100644 |
| --- a/drivers/tty/serial/8250/8250_omap.c |
| +++ b/drivers/tty/serial/8250/8250_omap.c |
| @@ -330,7 +330,7 @@ static void omap8250_restore_regs(struct uart_8250_port *up) |
| /* drop TCR + TLR access, we setup XON/XOFF later */ |
| serial8250_out_MCR(up, mcr); |
| |
| - serial_out(up, UART_IER, up->ier); |
| + serial8250_set_IER(up, up->ier); |
| |
| serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B); |
| serial_dl_write(up, priv->quot); |
| @@ -520,7 +520,7 @@ static void omap_8250_pm(struct uart_port *port, unsigned int state, |
| serial_out(up, UART_EFR, efr | UART_EFR_ECB); |
| serial_out(up, UART_LCR, 0); |
| |
| - serial_out(up, UART_IER, (state != 0) ? UART_IERX_SLEEP : 0); |
| + serial8250_set_IER(up, (state != 0) ? UART_IERX_SLEEP : 0); |
| serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B); |
| serial_out(up, UART_EFR, efr); |
| serial_out(up, UART_LCR, 0); |
| @@ -703,7 +703,7 @@ static int omap_8250_startup(struct uart_port *port) |
| goto err; |
| |
| up->ier = UART_IER_RLSI | UART_IER_RDI; |
| - serial_out(up, UART_IER, up->ier); |
| + serial8250_set_IER(up, up->ier); |
| |
| #ifdef CONFIG_PM |
| up->capabilities |= UART_CAP_RPM; |
| @@ -744,7 +744,7 @@ static void omap_8250_shutdown(struct uart_port *port) |
| serial_out(up, UART_OMAP_EFR2, 0x0); |
| |
| up->ier = 0; |
| - serial_out(up, UART_IER, 0); |
| + serial8250_set_IER(up, 0); |
| |
| if (up->dma) |
| serial8250_release_dma(up); |
| @@ -792,7 +792,7 @@ static void omap_8250_unthrottle(struct uart_port *port) |
| up->dma->rx_dma(up); |
| up->ier |= UART_IER_RLSI | UART_IER_RDI; |
| port->read_status_mask |= UART_LSR_DR; |
| - serial_out(up, UART_IER, up->ier); |
| + serial8250_set_IER(up, up->ier); |
| spin_unlock_irqrestore(&port->lock, flags); |
| |
| pm_runtime_mark_last_busy(port->dev); |
| @@ -883,7 +883,7 @@ static void __dma_rx_complete(void *param) |
| __dma_rx_do_complete(p); |
| if (!priv->throttled) { |
| p->ier |= UART_IER_RLSI | UART_IER_RDI; |
| - serial_out(p, UART_IER, p->ier); |
| + serial8250_set_IER(p, p->ier); |
| if (!(priv->habit & UART_HAS_EFR2)) |
| omap_8250_rx_dma(p); |
| } |
| @@ -940,7 +940,7 @@ static int omap_8250_rx_dma(struct uart_8250_port *p) |
| * callback to run. |
| */ |
| p->ier &= ~(UART_IER_RLSI | UART_IER_RDI); |
| - serial_out(p, UART_IER, p->ier); |
| + serial8250_set_IER(p, p->ier); |
| } |
| goto out; |
| } |
| @@ -1153,12 +1153,12 @@ static void am654_8250_handle_rx_dma(struct uart_8250_port *up, u8 iir, |
| * periodic timeouts, re-enable interrupts. |
| */ |
| up->ier &= ~(UART_IER_RLSI | UART_IER_RDI); |
| - serial_out(up, UART_IER, up->ier); |
| + serial8250_set_IER(up, up->ier); |
| omap_8250_rx_dma_flush(up); |
| serial_in(up, UART_IIR); |
| serial_out(up, UART_OMAP_EFR2, 0x0); |
| up->ier |= UART_IER_RLSI | UART_IER_RDI; |
| - serial_out(up, UART_IER, up->ier); |
| + serial8250_set_IER(up, up->ier); |
| } |
| } |
| |
| diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c |
| index 8efe31448df3..975c16267196 100644 |
| --- a/drivers/tty/serial/8250/8250_port.c |
| +++ b/drivers/tty/serial/8250/8250_port.c |
| @@ -744,7 +744,7 @@ static void serial8250_set_sleep(struct uart_8250_port *p, int sleep) |
| serial_out(p, UART_EFR, UART_EFR_ECB); |
| serial_out(p, UART_LCR, 0); |
| } |
| - serial_out(p, UART_IER, sleep ? UART_IERX_SLEEP : 0); |
| + serial8250_set_IER(p, sleep ? UART_IERX_SLEEP : 0); |
| if (p->capabilities & UART_CAP_EFR) { |
| serial_out(p, UART_LCR, UART_LCR_CONF_MODE_B); |
| serial_out(p, UART_EFR, efr); |
| @@ -755,12 +755,29 @@ static void serial8250_set_sleep(struct uart_8250_port *p, int sleep) |
| serial8250_rpm_put(p); |
| } |
| |
| -static void serial8250_clear_IER(struct uart_8250_port *up) |
| +static unsigned int serial8250_clear_IER(struct uart_8250_port *up) |
| { |
| + struct uart_port *port = &up->port; |
| + unsigned int clearval = 0; |
| + unsigned long flags; |
| + bool is_console; |
| + unsigned int prior; |
| + |
| + is_console = uart_console(port); |
| + |
| if (up->capabilities & UART_CAP_UUE) |
| - serial_out(up, UART_IER, UART_IER_UUE); |
| - else |
| - serial_out(up, UART_IER, 0); |
| + clearval = UART_IER_UUE; |
| + |
| + if (is_console) |
| + printk_cpu_sync_get_irqsave(flags); |
| + |
| + prior = serial_in(up, UART_IER); |
| + serial_out(up, UART_IER, clearval); |
| + |
| + if (is_console) |
| + printk_cpu_sync_put_irqrestore(flags); |
| + |
| + return prior; |
| } |
| |
| #ifdef CONFIG_SERIAL_8250_RSA |
| @@ -1026,8 +1043,11 @@ static int broken_efr(struct uart_8250_port *up) |
| */ |
| static void autoconfig_16550a(struct uart_8250_port *up) |
| { |
| + struct uart_port *port = &up->port; |
| unsigned char status1, status2; |
| unsigned int iersave; |
| + unsigned long flags; |
| + bool is_console; |
| |
| up->port.type = PORT_16550A; |
| up->capabilities |= UART_CAP_FIFO; |
| @@ -1139,6 +1159,11 @@ static void autoconfig_16550a(struct uart_8250_port *up) |
| return; |
| } |
| |
| + is_console = uart_console(port); |
| + |
| + if (is_console) |
| + printk_cpu_sync_get_irqsave(flags); |
| + |
| /* |
| * Try writing and reading the UART_IER_UUE bit (b6). |
| * If it works, this is probably one of the Xscale platform's |
| @@ -1174,6 +1199,9 @@ static void autoconfig_16550a(struct uart_8250_port *up) |
| } |
| serial_out(up, UART_IER, iersave); |
| |
| + if (is_console) |
| + printk_cpu_sync_put_irqrestore(flags); |
| + |
| /* |
| * We distinguish between 16550A and U6 16550A by counting |
| * how many bytes are in the FIFO. |
| @@ -1196,8 +1224,10 @@ static void autoconfig(struct uart_8250_port *up) |
| unsigned char status1, scratch, scratch2, scratch3; |
| unsigned char save_lcr, save_mcr; |
| struct uart_port *port = &up->port; |
| + unsigned long cs_flags; |
| unsigned long flags; |
| unsigned int old_capabilities; |
| + bool is_console; |
| |
| if (!port->iobase && !port->mapbase && !port->membase) |
| return; |
| @@ -1215,6 +1245,11 @@ static void autoconfig(struct uart_8250_port *up) |
| up->bugs = 0; |
| |
| if (!(port->flags & UPF_BUGGY_UART)) { |
| + is_console = uart_console(port); |
| + |
| + if (is_console) |
| + printk_cpu_sync_get_irqsave(cs_flags); |
| + |
| /* |
| * Do a simple existence test first; if we fail this, |
| * there's no point trying anything else. |
| @@ -1244,6 +1279,10 @@ static void autoconfig(struct uart_8250_port *up) |
| #endif |
| scratch3 = serial_in(up, UART_IER) & 0x0f; |
| serial_out(up, UART_IER, scratch); |
| + |
| + if (is_console) |
| + printk_cpu_sync_put_irqrestore(cs_flags); |
| + |
| if (scratch2 != 0 || scratch3 != 0x0F) { |
| /* |
| * We failed; there's nothing here |
| @@ -1367,7 +1406,9 @@ static void autoconfig_irq(struct uart_8250_port *up) |
| unsigned char save_mcr, save_ier; |
| unsigned char save_ICP = 0; |
| unsigned int ICP = 0; |
| + unsigned long flags; |
| unsigned long irqs; |
| + bool is_console; |
| int irq; |
| |
| if (port->flags & UPF_FOURPORT) { |
| @@ -1377,8 +1418,12 @@ static void autoconfig_irq(struct uart_8250_port *up) |
| inb_p(ICP); |
| } |
| |
| - if (uart_console(port)) |
| + is_console = uart_console(port); |
| + |
| + if (is_console) { |
| console_lock(); |
| + printk_cpu_sync_get_irqsave(flags); |
| + } |
| |
| /* forget possible initially masked and pending IRQ */ |
| probe_irq_off(probe_irq_on()); |
| @@ -1410,8 +1455,10 @@ static void autoconfig_irq(struct uart_8250_port *up) |
| if (port->flags & UPF_FOURPORT) |
| outb_p(save_ICP, ICP); |
| |
| - if (uart_console(port)) |
| + if (is_console) { |
| + printk_cpu_sync_put_irqrestore(flags); |
| console_unlock(); |
| + } |
| |
| port->irq = (irq > 0) ? irq : 0; |
| } |
| @@ -1424,7 +1471,7 @@ static void serial8250_stop_rx(struct uart_port *port) |
| |
| up->ier &= ~(UART_IER_RLSI | UART_IER_RDI); |
| up->port.read_status_mask &= ~UART_LSR_DR; |
| - serial_port_out(port, UART_IER, up->ier); |
| + serial8250_set_IER(up, up->ier); |
| |
| serial8250_rpm_put(up); |
| } |
| @@ -1454,7 +1501,7 @@ void serial8250_em485_stop_tx(struct uart_8250_port *p) |
| serial8250_clear_and_reinit_fifos(p); |
| |
| p->ier |= UART_IER_RLSI | UART_IER_RDI; |
| - serial_port_out(&p->port, UART_IER, p->ier); |
| + serial8250_set_IER(p, p->ier); |
| } |
| } |
| EXPORT_SYMBOL_GPL(serial8250_em485_stop_tx); |
| @@ -1703,7 +1750,7 @@ static void serial8250_disable_ms(struct uart_port *port) |
| mctrl_gpio_disable_ms(up->gpios); |
| |
| up->ier &= ~UART_IER_MSI; |
| - serial_port_out(port, UART_IER, up->ier); |
| + serial8250_set_IER(up, up->ier); |
| } |
| |
| static void serial8250_enable_ms(struct uart_port *port) |
| @@ -1719,7 +1766,7 @@ static void serial8250_enable_ms(struct uart_port *port) |
| up->ier |= UART_IER_MSI; |
| |
| serial8250_rpm_get(up); |
| - serial_port_out(port, UART_IER, up->ier); |
| + serial8250_set_IER(up, up->ier); |
| serial8250_rpm_put(up); |
| } |
| |
| @@ -2174,8 +2221,7 @@ static void serial8250_put_poll_char(struct uart_port *port, |
| /* |
| * First save the IER then disable the interrupts |
| */ |
| - ier = serial_port_in(port, UART_IER); |
| - serial8250_clear_IER(up); |
| + ier = serial8250_clear_IER(up); |
| |
| wait_for_xmitr(up, UART_LSR_BOTH_EMPTY); |
| /* |
| @@ -2188,7 +2234,7 @@ static void serial8250_put_poll_char(struct uart_port *port, |
| * and restore the IER |
| */ |
| wait_for_xmitr(up, UART_LSR_BOTH_EMPTY); |
| - serial_port_out(port, UART_IER, ier); |
| + serial8250_set_IER(up, ier); |
| serial8250_rpm_put(up); |
| } |
| |
| @@ -2197,8 +2243,10 @@ static void serial8250_put_poll_char(struct uart_port *port, |
| int serial8250_do_startup(struct uart_port *port) |
| { |
| struct uart_8250_port *up = up_to_u8250p(port); |
| + unsigned long cs_flags; |
| unsigned long flags; |
| unsigned char iir; |
| + bool is_console; |
| int retval; |
| u16 lsr; |
| |
| @@ -2219,7 +2267,7 @@ int serial8250_do_startup(struct uart_port *port) |
| up->acr = 0; |
| serial_port_out(port, UART_LCR, UART_LCR_CONF_MODE_B); |
| serial_port_out(port, UART_EFR, UART_EFR_ECB); |
| - serial_port_out(port, UART_IER, 0); |
| + serial8250_set_IER(up, 0); |
| serial_port_out(port, UART_LCR, 0); |
| serial_icr_write(up, UART_CSR, 0); /* Reset the UART */ |
| serial_port_out(port, UART_LCR, UART_LCR_CONF_MODE_B); |
| @@ -2229,7 +2277,7 @@ int serial8250_do_startup(struct uart_port *port) |
| |
| if (port->type == PORT_DA830) { |
| /* Reset the port */ |
| - serial_port_out(port, UART_IER, 0); |
| + serial8250_set_IER(up, 0); |
| serial_port_out(port, UART_DA830_PWREMU_MGMT, 0); |
| mdelay(10); |
| |
| @@ -2328,6 +2376,8 @@ int serial8250_do_startup(struct uart_port *port) |
| if (retval) |
| goto out; |
| |
| + is_console = uart_console(port); |
| + |
| if (port->irq && !(up->port.flags & UPF_NO_THRE_TEST)) { |
| unsigned char iir1; |
| |
| @@ -2344,6 +2394,9 @@ int serial8250_do_startup(struct uart_port *port) |
| */ |
| spin_lock_irqsave(&port->lock, flags); |
| |
| + if (is_console) |
| + printk_cpu_sync_get_irqsave(cs_flags); |
| + |
| wait_for_xmitr(up, UART_LSR_THRE); |
| serial_port_out_sync(port, UART_IER, UART_IER_THRI); |
| udelay(1); /* allow THRE to set */ |
| @@ -2354,6 +2407,9 @@ int serial8250_do_startup(struct uart_port *port) |
| iir = serial_port_in(port, UART_IIR); |
| serial_port_out(port, UART_IER, 0); |
| |
| + if (is_console) |
| + printk_cpu_sync_put_irqrestore(cs_flags); |
| + |
| spin_unlock_irqrestore(&port->lock, flags); |
| |
| if (port->irqflags & IRQF_SHARED) |
| @@ -2408,10 +2464,14 @@ int serial8250_do_startup(struct uart_port *port) |
| * Do a quick test to see if we receive an interrupt when we enable |
| * the TX irq. |
| */ |
| + if (is_console) |
| + printk_cpu_sync_get_irqsave(cs_flags); |
| serial_port_out(port, UART_IER, UART_IER_THRI); |
| lsr = serial_port_in(port, UART_LSR); |
| iir = serial_port_in(port, UART_IIR); |
| serial_port_out(port, UART_IER, 0); |
| + if (is_console) |
| + printk_cpu_sync_put_irqrestore(cs_flags); |
| |
| if (lsr & UART_LSR_TEMT && iir & UART_IIR_NO_INT) { |
| if (!(up->bugs & UART_BUG_TXEN)) { |
| @@ -2443,7 +2503,7 @@ int serial8250_do_startup(struct uart_port *port) |
| if (up->dma) { |
| const char *msg = NULL; |
| |
| - if (uart_console(port)) |
| + if (is_console) |
| msg = "forbid DMA for kernel console"; |
| else if (serial8250_request_dma(up)) |
| msg = "failed to request DMA"; |
| @@ -2494,7 +2554,7 @@ void serial8250_do_shutdown(struct uart_port *port) |
| */ |
| spin_lock_irqsave(&port->lock, flags); |
| up->ier = 0; |
| - serial_port_out(port, UART_IER, 0); |
| + serial8250_set_IER(up, 0); |
| spin_unlock_irqrestore(&port->lock, flags); |
| |
| synchronize_irq(port->irq); |
| @@ -2856,7 +2916,7 @@ serial8250_do_set_termios(struct uart_port *port, struct ktermios *termios, |
| if (up->capabilities & UART_CAP_RTOIE) |
| up->ier |= UART_IER_RTOIE; |
| |
| - serial_port_out(port, UART_IER, up->ier); |
| + serial8250_set_IER(up, up->ier); |
| |
| if (up->capabilities & UART_CAP_EFR) { |
| unsigned char efr = 0; |
| @@ -3321,7 +3381,7 @@ EXPORT_SYMBOL_GPL(serial8250_set_defaults); |
| |
| #ifdef CONFIG_SERIAL_8250_CONSOLE |
| |
| -static void serial8250_console_putchar(struct uart_port *port, unsigned char ch) |
| +static void serial8250_console_putchar_locked(struct uart_port *port, unsigned char ch) |
| { |
| struct uart_8250_port *up = up_to_u8250p(port); |
| |
| @@ -3329,6 +3389,18 @@ static void serial8250_console_putchar(struct uart_port *port, unsigned char ch) |
| serial_port_out(port, UART_TX, ch); |
| } |
| |
| +static void serial8250_console_putchar(struct uart_port *port, unsigned char ch) |
| +{ |
| + struct uart_8250_port *up = up_to_u8250p(port); |
| + unsigned long flags; |
| + |
| + wait_for_xmitr(up, UART_LSR_THRE); |
| + |
| + printk_cpu_sync_get_irqsave(flags); |
| + serial8250_console_putchar_locked(port, ch); |
| + printk_cpu_sync_put_irqrestore(flags); |
| +} |
| + |
| /* |
| * Restore serial console when h/w power-off detected |
| */ |
| @@ -3355,6 +3427,32 @@ static void serial8250_console_restore(struct uart_8250_port *up) |
| serial8250_out_MCR(up, up->mcr | UART_MCR_DTR | UART_MCR_RTS); |
| } |
| |
| +void serial8250_console_write_atomic(struct uart_8250_port *up, |
| + const char *s, unsigned int count) |
| +{ |
| + struct uart_port *port = &up->port; |
| + unsigned long flags; |
| + unsigned int ier; |
| + |
| + printk_cpu_sync_get_irqsave(flags); |
| + |
| + touch_nmi_watchdog(); |
| + |
| + ier = serial8250_clear_IER(up); |
| + |
| + if (atomic_fetch_inc(&up->console_printing)) { |
| + uart_console_write(port, "\n", 1, |
| + serial8250_console_putchar_locked); |
| + } |
| + uart_console_write(port, s, count, serial8250_console_putchar_locked); |
| + atomic_dec(&up->console_printing); |
| + |
| + wait_for_xmitr(up, UART_LSR_BOTH_EMPTY); |
| + serial8250_set_IER(up, ier); |
| + |
| + printk_cpu_sync_put_irqrestore(flags); |
| +} |
| + |
| /* |
| * Print a string to the serial port using the device FIFO |
| * |
| @@ -3400,20 +3498,15 @@ void serial8250_console_write(struct uart_8250_port *up, const char *s, |
| struct uart_port *port = &up->port; |
| unsigned long flags; |
| unsigned int ier, use_fifo; |
| - int locked = 1; |
| |
| touch_nmi_watchdog(); |
| |
| - if (oops_in_progress) |
| - locked = spin_trylock_irqsave(&port->lock, flags); |
| - else |
| - spin_lock_irqsave(&port->lock, flags); |
| + spin_lock_irqsave(&port->lock, flags); |
| |
| /* |
| * First save the IER then disable the interrupts |
| */ |
| - ier = serial_port_in(port, UART_IER); |
| - serial8250_clear_IER(up); |
| + ier = serial8250_clear_IER(up); |
| |
| /* check scratch reg to see if port powered off during system sleep */ |
| if (up->canary && (up->canary != serial_port_in(port, UART_SCR))) { |
| @@ -3447,10 +3540,12 @@ void serial8250_console_write(struct uart_8250_port *up, const char *s, |
| */ |
| !(up->port.flags & UPF_CONS_FLOW); |
| |
| + atomic_inc(&up->console_printing); |
| if (likely(use_fifo)) |
| serial8250_console_fifo_write(up, s, count); |
| else |
| uart_console_write(port, s, count, serial8250_console_putchar); |
| + atomic_dec(&up->console_printing); |
| |
| /* |
| * Finally, wait for transmitter to become empty |
| @@ -3463,8 +3558,7 @@ void serial8250_console_write(struct uart_8250_port *up, const char *s, |
| if (em485->tx_stopped) |
| up->rs485_stop_tx(up); |
| } |
| - |
| - serial_port_out(port, UART_IER, ier); |
| + serial8250_set_IER(up, ier); |
| |
| /* |
| * The receive handling will happen properly because the |
| @@ -3476,8 +3570,7 @@ void serial8250_console_write(struct uart_8250_port *up, const char *s, |
| if (up->msr_saved_flags) |
| serial8250_modem_status(up); |
| |
| - if (locked) |
| - spin_unlock_irqrestore(&port->lock, flags); |
| + spin_unlock_irqrestore(&port->lock, flags); |
| } |
| |
| static unsigned int probe_baud(struct uart_port *port) |
| @@ -3497,6 +3590,7 @@ static unsigned int probe_baud(struct uart_port *port) |
| |
| int serial8250_console_setup(struct uart_port *port, char *options, bool probe) |
| { |
| + struct uart_8250_port *up = up_to_u8250p(port); |
| int baud = 9600; |
| int bits = 8; |
| int parity = 'n'; |
| @@ -3506,6 +3600,8 @@ int serial8250_console_setup(struct uart_port *port, char *options, bool probe) |
| if (!port->iobase && !port->membase) |
| return -ENODEV; |
| |
| + atomic_set(&up->console_printing, 0); |
| + |
| if (options) |
| uart_parse_options(options, &baud, &parity, &bits, &flow); |
| else if (probe) |
| diff --git a/drivers/tty/serial/8250/Kconfig b/drivers/tty/serial/8250/Kconfig |
| index 583a340f9934..1f31320820a6 100644 |
| --- a/drivers/tty/serial/8250/Kconfig |
| +++ b/drivers/tty/serial/8250/Kconfig |
| @@ -9,6 +9,7 @@ config SERIAL_8250 |
| depends on !S390 |
| select SERIAL_CORE |
| select SERIAL_MCTRL_GPIO if GPIOLIB |
| + select HAVE_ATOMIC_CONSOLE |
| help |
| This selects whether you want to include the driver for the standard |
| serial ports. The standard answer is Y. People who might say N |
| diff --git a/drivers/tty/serial/amba-pl011.c b/drivers/tty/serial/amba-pl011.c |
| index 2f0f05259778..eee5141ecd3a 100644 |
| --- a/drivers/tty/serial/amba-pl011.c |
| +++ b/drivers/tty/serial/amba-pl011.c |
| @@ -2316,18 +2316,24 @@ pl011_console_write(struct console *co, const char *s, unsigned int count) |
| { |
| struct uart_amba_port *uap = amba_ports[co->index]; |
| unsigned int old_cr = 0, new_cr; |
| - unsigned long flags; |
| + unsigned long flags = 0; |
| int locked = 1; |
| |
| clk_enable(uap->clk); |
| |
| - local_irq_save(flags); |
| + /* |
| + * local_irq_save(flags); |
| + * |
| + * This local_irq_save() is nonsense. If we come in via sysrq |
| + * handling then interrupts are already disabled. Aside of |
| + * that the port.sysrq check is racy on SMP regardless. |
| + */ |
| if (uap->port.sysrq) |
| locked = 0; |
| else if (oops_in_progress) |
| - locked = spin_trylock(&uap->port.lock); |
| + locked = spin_trylock_irqsave(&uap->port.lock, flags); |
| else |
| - spin_lock(&uap->port.lock); |
| + spin_lock_irqsave(&uap->port.lock, flags); |
| |
| /* |
| * First save the CR then disable the interrupts |
| @@ -2353,8 +2359,7 @@ pl011_console_write(struct console *co, const char *s, unsigned int count) |
| pl011_write(old_cr, uap, REG_CR); |
| |
| if (locked) |
| - spin_unlock(&uap->port.lock); |
| - local_irq_restore(flags); |
| + spin_unlock_irqrestore(&uap->port.lock, flags); |
| |
| clk_disable(uap->clk); |
| } |
| diff --git a/drivers/tty/serial/omap-serial.c b/drivers/tty/serial/omap-serial.c |
| index beb7896ebf8a..ba5b843820c1 100644 |
| --- a/drivers/tty/serial/omap-serial.c |
| +++ b/drivers/tty/serial/omap-serial.c |
| @@ -1241,13 +1241,10 @@ serial_omap_console_write(struct console *co, const char *s, |
| unsigned int ier; |
| int locked = 1; |
| |
| - local_irq_save(flags); |
| - if (up->port.sysrq) |
| - locked = 0; |
| - else if (oops_in_progress) |
| - locked = spin_trylock(&up->port.lock); |
| + if (up->port.sysrq || oops_in_progress) |
| + locked = spin_trylock_irqsave(&up->port.lock, flags); |
| else |
| - spin_lock(&up->port.lock); |
| + spin_lock_irqsave(&up->port.lock, flags); |
| |
| /* |
| * First save the IER then disable the interrupts |
| @@ -1274,8 +1271,7 @@ serial_omap_console_write(struct console *co, const char *s, |
| check_modem_status(up); |
| |
| if (locked) |
| - spin_unlock(&up->port.lock); |
| - local_irq_restore(flags); |
| + spin_unlock_irqrestore(&up->port.lock, flags); |
| } |
| |
| static int __init |
| diff --git a/drivers/tty/sysrq.c b/drivers/tty/sysrq.c |
| index 248067197287..0db9dad8c99f 100644 |
| --- a/drivers/tty/sysrq.c |
| +++ b/drivers/tty/sysrq.c |
| @@ -582,6 +582,7 @@ void __handle_sysrq(int key, bool check_mask) |
| |
| rcu_sysrq_start(); |
| rcu_read_lock(); |
| + printk_prefer_direct_enter(); |
| /* |
| * Raise the apparent loglevel to maximum so that the sysrq header |
| * is shown to provide the user with positive feedback. We do not |
| @@ -623,6 +624,7 @@ void __handle_sysrq(int key, bool check_mask) |
| pr_cont("\n"); |
| console_loglevel = orig_log_level; |
| } |
| + printk_prefer_direct_exit(); |
| rcu_read_unlock(); |
| rcu_sysrq_end(); |
| |
| diff --git a/drivers/vdpa/vdpa_user/iova_domain.h b/drivers/vdpa/vdpa_user/iova_domain.h |
| index 4e0e50e7ac15..173e979b84a9 100644 |
| --- a/drivers/vdpa/vdpa_user/iova_domain.h |
| +++ b/drivers/vdpa/vdpa_user/iova_domain.h |
| @@ -14,7 +14,6 @@ |
| #include <linux/iova.h> |
| #include <linux/dma-mapping.h> |
| #include <linux/vhost_iotlb.h> |
| -#include <linux/rwlock.h> |
| |
| #define IOVA_START_PFN 1 |
| |
| diff --git a/include/linux/console.h b/include/linux/console.h |
| index 8c1686e2c233..8a813cbaf928 100644 |
| --- a/include/linux/console.h |
| +++ b/include/linux/console.h |
| @@ -16,6 +16,7 @@ |
| |
| #include <linux/atomic.h> |
| #include <linux/types.h> |
| +#include <linux/mutex.h> |
| |
| struct vc_data; |
| struct console_font_op; |
| @@ -137,9 +138,19 @@ static inline int con_debug_leave(void) |
| #define CON_BRL (32) /* Used for a braille device */ |
| #define CON_EXTENDED (64) /* Use the extended output format a la /dev/kmsg */ |
| |
| +#ifdef CONFIG_HAVE_ATOMIC_CONSOLE |
| +struct console_atomic_data { |
| + u64 seq; |
| + char *text; |
| + char *ext_text; |
| + char *dropped_text; |
| +}; |
| +#endif |
|