From e87d0f370660151296ec01c7e12d3063c2c8befb Mon Sep 17 00:00:00 2001 From: Joy Cho Date: Fri, 24 Aug 2018 15:52:31 +0900 Subject: [PATCH] ODROID-COMMON: add gator driver - gator driver version : 670 - default disabled Change-Id: Ibc67cfe17cb8d94ab8aa82c2bd2a16ba045be010 --- arch/arm64/configs/odroidn2_defconfig | 5 + drivers/Kconfig | 2 + drivers/Makefile | 1 + drivers/gator/COPYING | 339 ++++ drivers/gator/Kconfig | 66 + drivers/gator/Makefile | 107 ++ drivers/gator/gator.h | 143 ++ drivers/gator/gator_annotate.c | 189 +++ drivers/gator/gator_annotate_kernel.c | 200 +++ drivers/gator/gator_backtrace.c | 330 ++++ drivers/gator/gator_buffer.c | 171 ++ drivers/gator/gator_buffer_write.c | 83 + drivers/gator/gator_cookies.c | 550 ++++++ drivers/gator/gator_events_block.c | 162 ++ drivers/gator/gator_events_irq.c | 189 +++ drivers/gator/gator_events_l2c-310.c | 209 +++ drivers/gator/gator_events_mali_4xx.c | 624 +++++++ drivers/gator/gator_events_mali_4xx.h | 18 + drivers/gator/gator_events_mali_common.c | 72 + drivers/gator/gator_events_mali_common.h | 77 + drivers/gator/gator_events_mali_midgard.c | 570 +++++++ drivers/gator/gator_events_mali_midgard.h | 38 + drivers/gator/gator_events_mali_midgard_hw.c | 1026 +++++++++++ drivers/gator/gator_events_meminfo.c | 435 +++++ drivers/gator/gator_events_mmapped.c | 210 +++ drivers/gator/gator_events_net.c | 165 ++ drivers/gator/gator_events_perf_pmu.c | 581 +++++++ drivers/gator/gator_events_sched.c | 135 ++ drivers/gator/gator_fs.c | 375 ++++ drivers/gator/gator_hrtimer_gator.c | 80 + drivers/gator/gator_iks.c | 195 +++ drivers/gator/gator_main.c | 1501 +++++++++++++++++ drivers/gator/gator_marshaling.c | 379 +++++ drivers/gator/gator_pmu.c | 379 +++++ drivers/gator/gator_trace_gpu.c | 330 ++++ drivers/gator/gator_trace_power.c | 195 +++ drivers/gator/gator_trace_sched.c | 341 ++++ .../mali/mali_mjollnir_profiling_gator_api.h | 159 ++ .../mali/mali_utgard_profiling_gator_api.h | 197 +++ drivers/gator/mali_midgard.mk | 49 + 40 files changed, 10877 insertions(+) create mode 100644 drivers/gator/COPYING create mode 100644 drivers/gator/Kconfig create mode 100644 drivers/gator/Makefile create mode 100644 drivers/gator/gator.h create mode 100644 drivers/gator/gator_annotate.c create mode 100644 drivers/gator/gator_annotate_kernel.c create mode 100644 drivers/gator/gator_backtrace.c create mode 100644 drivers/gator/gator_buffer.c create mode 100644 drivers/gator/gator_buffer_write.c create mode 100644 drivers/gator/gator_cookies.c create mode 100644 drivers/gator/gator_events_block.c create mode 100644 drivers/gator/gator_events_irq.c create mode 100644 drivers/gator/gator_events_l2c-310.c create mode 100644 drivers/gator/gator_events_mali_4xx.c create mode 100644 drivers/gator/gator_events_mali_4xx.h create mode 100644 drivers/gator/gator_events_mali_common.c create mode 100644 drivers/gator/gator_events_mali_common.h create mode 100644 drivers/gator/gator_events_mali_midgard.c create mode 100644 drivers/gator/gator_events_mali_midgard.h create mode 100644 drivers/gator/gator_events_mali_midgard_hw.c create mode 100644 drivers/gator/gator_events_meminfo.c create mode 100644 drivers/gator/gator_events_mmapped.c create mode 100644 drivers/gator/gator_events_net.c create mode 100644 drivers/gator/gator_events_perf_pmu.c create mode 100644 drivers/gator/gator_events_sched.c create mode 100644 drivers/gator/gator_fs.c create mode 100644 drivers/gator/gator_hrtimer_gator.c create mode 100644 drivers/gator/gator_iks.c create mode 100644 drivers/gator/gator_main.c create mode 100644 drivers/gator/gator_marshaling.c create mode 100644 drivers/gator/gator_pmu.c create mode 100644 drivers/gator/gator_trace_gpu.c create mode 100644 drivers/gator/gator_trace_power.c create mode 100644 drivers/gator/gator_trace_sched.c create mode 100644 drivers/gator/mali/mali_mjollnir_profiling_gator_api.h create mode 100644 drivers/gator/mali/mali_utgard_profiling_gator_api.h create mode 100644 drivers/gator/mali_midgard.mk diff --git a/arch/arm64/configs/odroidn2_defconfig b/arch/arm64/configs/odroidn2_defconfig index 62c41ecce0fa..fae3e6533e72 100644 --- a/arch/arm64/configs/odroidn2_defconfig +++ b/arch/arm64/configs/odroidn2_defconfig @@ -4518,6 +4518,11 @@ CONFIG_MALI_EXPERT=y # CONFIG_MALI_PWRSOFT_765 is not set CONFIG_MALI_KUTF=m +# +# Gator module for ARM streamline +# +# CONFIG_GATOR is not set + # # Firmware Drivers # diff --git a/drivers/Kconfig b/drivers/Kconfig index b62cf1aec34d..e8521bcf82aa 100644 --- a/drivers/Kconfig +++ b/drivers/Kconfig @@ -207,4 +207,6 @@ source "drivers/fpga/Kconfig" source "drivers/tee/Kconfig" source "drivers/gpu/arm/Kconfig" + +source "drivers/gator/Kconfig" endmenu diff --git a/drivers/Makefile b/drivers/Makefile index 1f05f0e6440a..f3712c2d27bc 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -177,3 +177,4 @@ obj-$(CONFIG_NVMEM) += nvmem/ obj-$(CONFIG_FPGA) += fpga/ obj-$(CONFIG_TEE) += tee/ obj-$(CONFIG_AMLOGIC_DRIVER) += amlogic/ +obj-$(CONFIG_GATOR) += gator/ diff --git a/drivers/gator/COPYING b/drivers/gator/COPYING new file mode 100644 index 000000000000..d159169d1050 --- /dev/null +++ b/drivers/gator/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/drivers/gator/Kconfig b/drivers/gator/Kconfig new file mode 100644 index 000000000000..84f48be6bf48 --- /dev/null +++ b/drivers/gator/Kconfig @@ -0,0 +1,66 @@ +menu "Gator module for ARM streamline" + +config GATOR + tristate "Gator module for Arm's Streamline Performance Analyzer" + default m if (ARM || ARM64) + depends on PROFILING + depends on HIGH_RES_TIMERS + depends on LOCAL_TIMERS || !(ARM && SMP) + depends on PERF_EVENTS + depends on HW_PERF_EVENTS || !(ARM || ARM64) + select TRACING + help + Gator module for Arm's Streamline Performance Analyzer + +config GATOR_DO_NOT_ONLINE_CORES_AT_STARTUP + bool "Disable onlining all CPU cores by default when module loads" + default n + depends on GATOR + help + Prevent gator from online all CPU cores when it first loads. + The default option is to online all CPU cores when gator first + loads. This is safe and necessary in most cases as it allows gator to correctly + identify all cores in the system. + Most users should leave this option disabled. + Even if the option is left disabled, it is possible to disable onlining + behaviour at module load time by setting the disable_cpu_onlining + module parameter to true. + +config GATOR_WITH_MALI_SUPPORT + bool + depends on GATOR + +choice + prompt "Enable Mali GPU support in Gator" + depends on GATOR + optional + help + Enable Mali GPU support in Gator + +config GATOR_MALI_4XXMP + bool "Mali-400MP or Mali-450MP" + select GATOR_WITH_MALI_SUPPORT + +config GATOR_MALI_MIDGARD + bool "Mali-T60x, Mali-T62x, Mali-T72x or Mali-T76x" + select GATOR_WITH_MALI_SUPPORT + +endchoice + +config GATOR_MALI_4XXMP_PATH + string "Path to Mali driver" + depends on GATOR_MALI_4XXMP + default "drivers/gpu/arm/mali400mp" + help + The gator code adds this to its include path so it can get the Mali + trace headers with: #include "linux/mali_linux_trace.h" + +config GATOR_MALI_MIDGARD_PATH + string "Path to Mali driver" + depends on GATOR_MALI_MIDGARD + default "drivers/gpu/arm/midgard" + help + The gator code adds this to its include path so it can get the Mali + trace headers with: #include "linux/mali_linux_trace.h" + +endmenu diff --git a/drivers/gator/Makefile b/drivers/gator/Makefile new file mode 100644 index 000000000000..fff86036beed --- /dev/null +++ b/drivers/gator/Makefile @@ -0,0 +1,107 @@ +ifneq ($(KERNELRELEASE),) + +# Uncomment the following line to enable kernel stack unwinding within gator, or update gator_backtrace.c +# EXTRA_CFLAGS += -DGATOR_KERNEL_STACK_UNWINDING + +CONFIG_GATOR ?= m +obj-$(CONFIG_GATOR) := gator.o + +gator-y := gator_main.o \ + gator_events_block.o \ + gator_events_irq.o \ + gator_events_meminfo.o \ + gator_events_mmapped.o \ + gator_events_net.o \ + gator_events_sched.o + + chk_events.h = : + quiet_chk_events.h = echo ' CHK $@' +silent_chk_events.h = : + +# Convert the old GATOR_WITH_MALI_SUPPORT to the new kernel flags +ifneq ($(GATOR_WITH_MALI_SUPPORT),) + CONFIG_GATOR_WITH_MALI_SUPPORT := y + ifeq ($(GATOR_WITH_MALI_SUPPORT),MALI_MIDGARD) + CONFIG_GATOR_MALI_4XXMP := n + CONFIG_GATOR_MALI_MIDGARD := y + EXTRA_CFLAGS += -DMALI_SUPPORT=MALI_MIDGARD_OR_BIFROST + else + ifeq ($(GATOR_WITH_MALI_SUPPORT),MALI_BIFROST) + CONFIG_GATOR_MALI_4XXMP := n + CONFIG_GATOR_MALI_MIDGARD := y + EXTRA_CFLAGS += -DMALI_SUPPORT=MALI_MIDGARD_OR_BIFROST + else + # Neither Midgard, nor Bifrost, must be 4xx then + CONFIG_GATOR_MALI_4XXMP := y + CONFIG_GATOR_MALI_MIDGARD := n + EXTRA_CFLAGS += -DMALI_SUPPORT=$(GATOR_WITH_MALI_SUPPORT) + endif + endif + ifneq ($(GATOR_MALI_INTERFACE_STYLE),) + EXTRA_CFLAGS += -DGATOR_MALI_INTERFACE_STYLE=$(GATOR_MALI_INTERFACE_STYLE) + endif + ifneq ($(GATOR_MALI_4XXMP_PATH),) + CONFIG_GATOR_MALI_4XXMP_PATH = $(GATOR_MALI_4XXMP_PATH) + endif + CONFIG_GATOR_MALI_4XXMP_PATH ?= drivers/gpu/arm/mali400mp + ifneq ($(GATOR_MALI_MIDGARD_PATH),) + CONFIG_GATOR_MALI_MIDGARD_PATH = $(GATOR_MALI_MIDGARD_PATH) + endif + CONFIG_GATOR_MALI_MIDGARD_PATH ?= drivers/gpu/arm/midgard +endif + +ifeq ($(CONFIG_GATOR_WITH_MALI_SUPPORT),y) + ifeq ($(CONFIG_GATOR_MALI_MIDGARD),y) + gator-y += gator_events_mali_midgard.o \ + gator_events_mali_midgard_hw.o + include $(src)/mali_midgard.mk + else + gator-y += gator_events_mali_4xx.o + endif + gator-y += gator_events_mali_common.o + + ifneq ($(CONFIG_GATOR_MALI_4XXMP_PATH),) + ccflags-$(CONFIG_GATOR_MALI_4XXMP) += -I$(CONFIG_GATOR_MALI_4XXMP_PATH) + endif + ifneq ($(CONFIG_GATOR_MALI_MIDGARD_PATH),) + ccflags-$(CONFIG_GATOR_MALI_MIDGARD) += -I$(CONFIG_GATOR_MALI_MIDGARD_PATH) + endif + ccflags-$(CONFIG_GATOR_MALI_4XXMP) += -DMALI_SUPPORT=MALI_4xx + ccflags-$(CONFIG_GATOR_MALI_MIDGARD) += -DMALI_SUPPORT=MALI_MIDGARD_OR_BIFROST +endif + +gator-additional-dependencies := $(obj)/generated_gator_src_md5.h +clean-files := generated_gator_src_md5.h + +# Should the original or new block_rq_complete API be used? +OLD_BLOCK_RQ_COMPLETE := $(shell grep -A3 block_rq_complete $(srctree)/include/trace/events/block.h | grep nr_bytes -q; echo $$?) +EXTRA_CFLAGS += -DOLD_BLOCK_RQ_COMPLETE=$(OLD_BLOCK_RQ_COMPLETE) + +gator-$(CONFIG_ARM) += gator_events_l2c-310.o + +gator-$(CONFIG_ARM64) += + +$(obj)/gator_main.o: $(gator-additional-dependencies) + +# Note, in the recipe below we use "cd $(srctree) && cd $(src)" rather than +# "cd $(srctree)/$(src)" because under DKMS $(src) is an absolute path, and we +# can't just use $(src) because for normal kernel builds this is relative to +# $(srctree) +$(obj)/generated_gator_src_md5.h: $(wildcard $(src)/gator_*.h $(src)/gator_*.c $(src)/mali/*.h) + @$($(quiet)chk_events.h) + $(Q)cd $(srctree) && cd $(src) ; $(CONFIG_SHELL) -c "echo 'static char * gator_src_md5 = \"'\`ls gator_*.c gator_*.h mali/*.h | grep -Ev '^(generated_gator_src_md5\.h|gator\.mod\.c)$$' | LC_ALL=C sort | xargs cat | md5sum | cut -b 1-32\`'\";'" > $(abspath $@) + +else + +all: + @echo + @echo "usage:" + @echo " make -C M=\`pwd\` ARCH=arm CROSS_COMPILE=<...> modules" + @echo + $(error) + +clean: + rm -f *.o .*.cmd generated_gator_src_md5.h modules.order Module.symvers gator.ko gator.mod.c + rm -rf .tmp_versions + +endif diff --git a/drivers/gator/gator.h b/drivers/gator/gator.h new file mode 100644 index 000000000000..53cdb5e23c3b --- /dev/null +++ b/drivers/gator/gator.h @@ -0,0 +1,143 @@ +/** + * Copyright (C) Arm Limited 2010-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef GATOR_H_ +#define GATOR_H_ + +#include +#include +#include +#include + +#define GATOR_PERF_PMU_SUPPORT (defined(CONFIG_PERF_EVENTS) && (!(defined(__arm__) || defined(__aarch64__)) || defined(CONFIG_HW_PERF_EVENTS))) +#define GATOR_CPU_FREQ_SUPPORT defined(CONFIG_CPU_FREQ) +#define GATOR_IKS_SUPPORT defined(CONFIG_BL_SWITCHER) + +/* cpu ids */ +#define CORTEX_A5 0x41c05 +#define CORTEX_A9 0x41c09 +#define OTHER 0xfffff + +#define MAXSIZE_CORE_NAME 32 + +struct gator_cpu { + struct list_head list; + unsigned long cpuid; + unsigned long pmnc_counters; + /* Human readable name */ + char core_name[MAXSIZE_CORE_NAME]; + /* gatorfs event and Perf PMU name */ + char pmnc_name[MAXSIZE_CORE_NAME]; + /* compatible from Documentation/devicetree/bindings/arm/cpus.txt */ + char dt_name[MAXSIZE_CORE_NAME]; +}; + +/* clusters */ +#define GATOR_CLUSTER_COUNT 4 + +extern const struct gator_cpu *gator_clusters[GATOR_CLUSTER_COUNT]; +extern int gator_clusterids[NR_CPUS]; +extern int gator_cluster_count; + +/* gpu enums */ +#define MALI_4xx 1 +#define MALI_MIDGARD_OR_BIFROST 2 + +/****************************************************************************** + * Filesystem + ******************************************************************************/ +struct dentry *gatorfs_mkdir(struct super_block *sb, struct dentry *root, + char const *name); + +int gatorfs_create_ulong(struct super_block *sb, struct dentry *root, + char const *name, unsigned long *val); + +int gatorfs_create_ro_ulong(struct super_block *sb, struct dentry *root, + char const *name, unsigned long *val); + +/****************************************************************************** + * Tracepoints + ******************************************************************************/ +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0) +# error Kernels prior to 3.4 not supported. DS-5 v5.21 and earlier supported 2.6.32 and later. +#elif LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0) +# define GATOR_DEFINE_PROBE(probe_name, proto) \ + static void probe_##probe_name(void *data, PARAMS(proto)) +# define GATOR_REGISTER_TRACE(probe_name) \ + register_trace_##probe_name(probe_##probe_name, NULL) +# define GATOR_UNREGISTER_TRACE(probe_name) \ + unregister_trace_##probe_name(probe_##probe_name, NULL) +#else +# define GATOR_DEFINE_PROBE(probe_name, proto) \ + extern struct tracepoint *gator_tracepoint_##probe_name; \ + static void probe_##probe_name(void *data, PARAMS(proto)) +# define GATOR_REGISTER_TRACE(probe_name) \ + ((gator_tracepoint_##probe_name == NULL) || tracepoint_probe_register(gator_tracepoint_##probe_name, probe_##probe_name, NULL)) +# define GATOR_UNREGISTER_TRACE(probe_name) \ + tracepoint_probe_unregister(gator_tracepoint_##probe_name, probe_##probe_name, NULL) +#endif + +/****************************************************************************** + * Events + ******************************************************************************/ +struct gator_interface { + const char *const name; + /* Complementary function to init */ + void (*shutdown)(void); + int (*create_files)(struct super_block *sb, struct dentry *root); + int (*start)(void); + /* Complementary function to start */ + void (*stop)(void); + int (*online)(int **buffer, bool migrate); + int (*offline)(int **buffer, bool migrate); + /* called in process context but may not be running on core 'cpu' */ + void (*online_dispatch)(int cpu, bool migrate); + /* called in process context but may not be running on core 'cpu' */ + void (*offline_dispatch)(int cpu, bool migrate); + int (*read)(int **buffer, bool sched_switch); + int (*read64)(long long **buffer, bool sched_switch); + int (*read_proc)(long long **buffer, struct task_struct *); + struct list_head list; +}; + +u64 gator_get_time(void); +int gator_events_install(struct gator_interface *interface); +int gator_events_get_key(void); +u32 gator_cpuid(void); + +void gator_marshal_activity_switch(int core, int key, int activity, int pid); + +#if !GATOR_IKS_SUPPORT + +#define get_physical_cpu() smp_processor_id() +#define lcpu_to_pcpu(lcpu) lcpu +#define pcpu_to_lcpu(pcpu) pcpu + +#else + +#define get_physical_cpu() lcpu_to_pcpu(get_logical_cpu()) +int lcpu_to_pcpu(const int lcpu); +int pcpu_to_lcpu(const int pcpu); + +#endif + +#define get_logical_cpu() smp_processor_id() +#define on_primary_core() (get_logical_cpu() == 0) + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 15, 0) +# define setup_deferrable_timer_on_stack(TIMER, HANDLER, VALUE) \ + timer_setup((TIMER), (HANDLER), TIMER_DEFERRABLE) +# define destroy_timer_on_stack(TIMER) +# define DECLARE_TIMER_HANDLER(NAME) \ + void NAME (struct timer_list * timers) +#else +# define DECLARE_TIMER_HANDLER(NAME) \ + void NAME (unsigned long arg) +#endif + +#endif /* GATOR_H_ */ diff --git a/drivers/gator/gator_annotate.c b/drivers/gator/gator_annotate.c new file mode 100644 index 000000000000..38058743cdb8 --- /dev/null +++ b/drivers/gator/gator_annotate.c @@ -0,0 +1,189 @@ +/** + * Copyright (C) Arm Limited 2010-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#include +#include +#include +#include +#include +#include +#include + +static DEFINE_SPINLOCK(annotate_lock); +static bool collect_annotations; + +static int annotate_copy(struct file *file, char const __user *buf, size_t count) +{ + int cpu = 0; + int write = per_cpu(gator_buffer_write, cpu)[ANNOTATE_BUF]; + + if (file == NULL) { + /* copy from kernel */ + memcpy(&per_cpu(gator_buffer, cpu)[ANNOTATE_BUF][write], buf, count); + } else { + /* copy from user space */ + if (copy_from_user(&per_cpu(gator_buffer, cpu)[ANNOTATE_BUF][write], buf, count) != 0) + return -1; + } + per_cpu(gator_buffer_write, cpu)[ANNOTATE_BUF] = (write + count) & gator_buffer_mask[ANNOTATE_BUF]; + + return 0; +} + +static ssize_t annotate_write(struct file *file, char const __user *buf, size_t count_orig, loff_t *offset) +{ + int pid, cpu, header_size, available, contiguous, length1, length2, size, count = count_orig & 0x7fffffff; + bool interrupt_context; + + if (*offset) + return -EINVAL; + + interrupt_context = in_interrupt(); + /* Annotations are not supported in interrupt context, but may work + * if you comment out the the next four lines of code. By doing so, + * annotations in interrupt context can result in deadlocks and lost + * data. + */ + if (interrupt_context) { + pr_warning("gator: Annotations are not supported in interrupt context. Edit gator_annotate.c in the gator driver to enable annotations in interrupt context.\n"); + return -EINVAL; + } + + retry: + /* synchronize between cores and with collect_annotations */ + spin_lock(&annotate_lock); + + if (!collect_annotations) { + /* Not collecting annotations, tell the caller everything was written */ + size = count_orig; + goto annotate_write_out; + } + + /* Annotation only uses a single per-cpu buffer as the data must be in order to the engine */ + cpu = 0; + + if (current == NULL) + pid = 0; + else + pid = current->pid; + + /* determine total size of the payload */ + header_size = MAXSIZE_PACK32 * 3 + MAXSIZE_PACK64; + available = buffer_bytes_available(cpu, ANNOTATE_BUF) - header_size; + size = count < available ? count : available; + + if (size <= 0) { + /* Buffer is full, wait until space is available */ + spin_unlock(&annotate_lock); + + /* Drop the annotation as blocking is not allowed in interrupt context */ + if (interrupt_context) + return -EINVAL; + + wait_event_interruptible(gator_annotate_wait, buffer_bytes_available(cpu, ANNOTATE_BUF) > header_size || !collect_annotations); + + /* Check to see if a signal is pending */ + if (signal_pending(current)) + return -EINTR; + + goto retry; + } + + /* synchronize shared variables annotateBuf and annotatePos */ + if (per_cpu(gator_buffer, cpu)[ANNOTATE_BUF]) { + u64 time = gator_get_time(); + + gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, get_physical_cpu()); + gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, pid); + gator_buffer_write_packed_int64(cpu, ANNOTATE_BUF, time); + gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, size); + + /* determine the sizes to capture, length1 + length2 will equal size */ + contiguous = contiguous_space_available(cpu, ANNOTATE_BUF); + if (size < contiguous) { + length1 = size; + length2 = 0; + } else { + length1 = contiguous; + length2 = size - contiguous; + } + + if (annotate_copy(file, buf, length1) != 0) { + size = -EINVAL; + goto annotate_write_out; + } + + if (length2 > 0 && annotate_copy(file, &buf[length1], length2) != 0) { + size = -EINVAL; + goto annotate_write_out; + } + + /* Check and commit; commit is set to occur once buffer is 3/4 full */ + buffer_check(cpu, ANNOTATE_BUF, time); + } + +annotate_write_out: + spin_unlock(&annotate_lock); + + /* return the number of bytes written */ + return size; +} + +#include "gator_annotate_kernel.c" + +static int annotate_release(struct inode *inode, struct file *file) +{ + int cpu = 0; + + /* synchronize between cores */ + spin_lock(&annotate_lock); + + if (per_cpu(gator_buffer, cpu)[ANNOTATE_BUF] && buffer_check_space(cpu, ANNOTATE_BUF, MAXSIZE_PACK64 + 3 * MAXSIZE_PACK32)) { + uint32_t pid = current->pid; + + gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, get_physical_cpu()); + gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, pid); + /* time */ + gator_buffer_write_packed_int64(cpu, ANNOTATE_BUF, 0); + /* size */ + gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, 0); + } + + /* Check and commit; commit is set to occur once buffer is 3/4 full */ + buffer_check(cpu, ANNOTATE_BUF, gator_get_time()); + + spin_unlock(&annotate_lock); + + return 0; +} + +static const struct file_operations annotate_fops = { + .write = annotate_write, + .release = annotate_release +}; + +static int gator_annotate_create_files(struct super_block *sb, struct dentry *root) +{ + return gatorfs_create_file_perm(sb, root, "annotate", &annotate_fops, 0666); +} + +static int gator_annotate_start(void) +{ + collect_annotations = true; + return 0; +} + +static void gator_annotate_stop(void) +{ + /* the spinlock here will ensure that when this function exits, we are not in the middle of an annotation */ + spin_lock(&annotate_lock); + collect_annotations = false; + wake_up(&gator_annotate_wait); + spin_unlock(&annotate_lock); +} diff --git a/drivers/gator/gator_annotate_kernel.c b/drivers/gator/gator_annotate_kernel.c new file mode 100644 index 000000000000..d4a8195ad96f --- /dev/null +++ b/drivers/gator/gator_annotate_kernel.c @@ -0,0 +1,200 @@ +/** + * Copyright (C) Arm Limited 2012-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#define ESCAPE_CODE 0x1c +#define STRING_ANNOTATION 0x06 +#define NAME_CHANNEL_ANNOTATION 0x07 +#define NAME_GROUP_ANNOTATION 0x08 +#define VISUAL_ANNOTATION 0x04 +#define MARKER_ANNOTATION 0x05 + +static void kannotate_write(const char *ptr, unsigned int size) +{ + int retval; + int pos = 0; + loff_t offset = 0; + + while (pos < size) { + retval = annotate_write(NULL, &ptr[pos], size - pos, &offset); + if (retval < 0) { + pr_warning("gator: kannotate_write failed with return value %d\n", retval); + return; + } + pos += retval; + } +} + +static void marshal_u16(char *buf, u16 val) +{ + buf[0] = val & 0xff; + buf[1] = (val >> 8) & 0xff; +} + +static void marshal_u32(char *buf, u32 val) +{ + buf[0] = val & 0xff; + buf[1] = (val >> 8) & 0xff; + buf[2] = (val >> 16) & 0xff; + buf[3] = (val >> 24) & 0xff; +} + +void gator_annotate_channel(int channel, const char *str) +{ + const u16 str_size = strlen(str) & 0xffff; + char header[8]; + + header[0] = ESCAPE_CODE; + header[1] = STRING_ANNOTATION; + marshal_u32(header + 2, channel); + marshal_u16(header + 6, str_size); + kannotate_write(header, sizeof(header)); + kannotate_write(str, str_size); +} +EXPORT_SYMBOL(gator_annotate_channel); + +void gator_annotate(const char *str) +{ + gator_annotate_channel(0, str); +} +EXPORT_SYMBOL(gator_annotate); + +void gator_annotate_channel_color(int channel, int color, const char *str) +{ + const u16 str_size = (strlen(str) + 4) & 0xffff; + char header[12]; + + header[0] = ESCAPE_CODE; + header[1] = STRING_ANNOTATION; + marshal_u32(header + 2, channel); + marshal_u16(header + 6, str_size); + marshal_u32(header + 8, color); + kannotate_write(header, sizeof(header)); + kannotate_write(str, str_size - 4); +} +EXPORT_SYMBOL(gator_annotate_channel_color); + +void gator_annotate_color(int color, const char *str) +{ + gator_annotate_channel_color(0, color, str); +} +EXPORT_SYMBOL(gator_annotate_color); + +void gator_annotate_channel_end(int channel) +{ + char header[8]; + + header[0] = ESCAPE_CODE; + header[1] = STRING_ANNOTATION; + marshal_u32(header + 2, channel); + marshal_u16(header + 6, 0); + kannotate_write(header, sizeof(header)); +} +EXPORT_SYMBOL(gator_annotate_channel_end); + +void gator_annotate_end(void) +{ + gator_annotate_channel_end(0); +} +EXPORT_SYMBOL(gator_annotate_end); + +void gator_annotate_name_channel(int channel, int group, const char *str) +{ + const u16 str_size = strlen(str) & 0xffff; + char header[12]; + + header[0] = ESCAPE_CODE; + header[1] = NAME_CHANNEL_ANNOTATION; + marshal_u32(header + 2, channel); + marshal_u32(header + 6, group); + marshal_u16(header + 10, str_size); + kannotate_write(header, sizeof(header)); + kannotate_write(str, str_size); +} +EXPORT_SYMBOL(gator_annotate_name_channel); + +void gator_annotate_name_group(int group, const char *str) +{ + const u16 str_size = strlen(str) & 0xffff; + char header[8]; + + header[0] = ESCAPE_CODE; + header[1] = NAME_GROUP_ANNOTATION; + marshal_u32(header + 2, group); + marshal_u16(header + 6, str_size); + kannotate_write(header, sizeof(header)); + kannotate_write(str, str_size); +} +EXPORT_SYMBOL(gator_annotate_name_group); + +void gator_annotate_visual(const char *data, unsigned int length, const char *str) +{ + const u16 str_size = strlen(str) & 0xffff; + char header[4]; + char header_length[4]; + + header[0] = ESCAPE_CODE; + header[1] = VISUAL_ANNOTATION; + marshal_u16(header + 2, str_size); + marshal_u32(header_length, length); + kannotate_write(header, sizeof(header)); + kannotate_write(str, str_size); + kannotate_write(header_length, sizeof(header_length)); + kannotate_write(data, length); +} +EXPORT_SYMBOL(gator_annotate_visual); + +void gator_annotate_marker(void) +{ + char header[4]; + + header[0] = ESCAPE_CODE; + header[1] = MARKER_ANNOTATION; + marshal_u16(header + 2, 0); + kannotate_write(header, sizeof(header)); +} +EXPORT_SYMBOL(gator_annotate_marker); + +void gator_annotate_marker_str(const char *str) +{ + const u16 str_size = strlen(str) & 0xffff; + char header[4]; + + header[0] = ESCAPE_CODE; + header[1] = MARKER_ANNOTATION; + marshal_u16(header + 2, str_size); + kannotate_write(header, sizeof(header)); + kannotate_write(str, str_size); +} +EXPORT_SYMBOL(gator_annotate_marker_str); + +void gator_annotate_marker_color(int color) +{ + char header[8]; + + header[0] = ESCAPE_CODE; + header[1] = MARKER_ANNOTATION; + marshal_u16(header + 2, 4); + marshal_u32(header + 4, color); + kannotate_write(header, sizeof(header)); +} +EXPORT_SYMBOL(gator_annotate_marker_color); + +void gator_annotate_marker_color_str(int color, const char *str) +{ + const u16 str_size = (strlen(str) + 4) & 0xffff; + char header[8]; + + header[0] = ESCAPE_CODE; + header[1] = MARKER_ANNOTATION; + marshal_u16(header + 2, str_size); + marshal_u32(header + 4, color); + kannotate_write(header, sizeof(header)); + kannotate_write(str, str_size - 4); +} +EXPORT_SYMBOL(gator_annotate_marker_color_str); diff --git a/drivers/gator/gator_backtrace.c b/drivers/gator/gator_backtrace.c new file mode 100644 index 000000000000..f5e96e09839e --- /dev/null +++ b/drivers/gator/gator_backtrace.c @@ -0,0 +1,330 @@ +/** + * Copyright (C) Arm Limited 2010-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#if defined(__arm__) || defined(__aarch64__) +#define GATOR_KERNEL_UNWINDING 1 +#define GATOR_USER_UNWINDING 1 +#else +#define GATOR_KERNEL_UNWINDING 0 +#define GATOR_USER_UNWINDING 0 +#endif + +/* on 4.9 walk_stackframe was unexported so use save_stack_trace instead */ +#if defined(MODULE) && defined(__aarch64__) && (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0)) +#define GATOR_KERNEL_UNWINDING_USE_WALK_STACKFRAME 0 +#else +#define GATOR_KERNEL_UNWINDING_USE_WALK_STACKFRAME 1 +#endif + +#if (!GATOR_KERNEL_UNWINDING_USE_WALK_STACKFRAME) && !defined(CONFIG_STACKTRACE) +#error "CONFIG_STACKTRACE is required for kernel unwinding" +#endif + +/* Uncomment the following line to enable kernel stack unwinding within gator, note it can also be defined from the Makefile */ +/* #define GATOR_KERNEL_STACK_UNWINDING */ + +#if GATOR_KERNEL_UNWINDING + +#include +#if !GATOR_KERNEL_UNWINDING_USE_WALK_STACKFRAME +#include +#endif + +#if !defined(GATOR_KERNEL_STACK_UNWINDING) +/* Disabled by default */ +MODULE_PARM_DESC(kernel_stack_unwinding, "Allow kernel stack unwinding."); +static bool kernel_stack_unwinding; +module_param(kernel_stack_unwinding, bool, 0644); +#endif + +#if GATOR_KERNEL_UNWINDING_USE_WALK_STACKFRAME + +/* -------------------------- KERNEL UNWINDING USING walk_stackframe -------------------------- */ + +static int report_trace(struct stackframe *frame, void *d) +{ + unsigned int *depth = d, cookie = NO_COOKIE; + unsigned long addr = frame->pc; + + if (*depth) { +#if defined(MODULE) + unsigned int cpu = get_physical_cpu(); + struct module *mod = __module_address(addr); + + if (mod) { + cookie = get_cookie(cpu, current, mod->name, false); + addr = addr - +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0) + (unsigned long)mod->module_core; +#else + (unsigned long)mod->core_layout.base; +#endif + } + else { + cookie = NO_COOKIE; + } +#endif + marshal_backtrace(addr & ~1, cookie, 1); + (*depth)--; + } + + return *depth == 0; +} + +static void kernel_backtrace(int cpu, struct pt_regs *const regs) +{ +#ifdef GATOR_KERNEL_STACK_UNWINDING + int depth = gator_backtrace_depth; +#else + int depth = (kernel_stack_unwinding ? gator_backtrace_depth : 1); +#endif + struct stackframe frame; + + if (depth == 0) + depth = 1; +#if defined(__arm__) + frame.fp = regs->ARM_fp; + frame.sp = regs->ARM_sp; + frame.lr = regs->ARM_lr; + frame.pc = regs->ARM_pc; +#else + frame.fp = regs->regs[29]; + frame.sp = regs->sp; + frame.pc = regs->pc; +#endif + +#if defined(__aarch64__) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0) + walk_stackframe(current, &frame, report_trace, &depth); +#else + walk_stackframe(&frame, report_trace, &depth); +#endif +} + +#else /* GATOR_KERNEL_UNWINDING_USE_WALK_STACKFRAME */ + +/* -------------------------- KERNEL UNWINDING USING save_stack_trace_regs -------------------------- */ + +#define GATOR_KMOD_STACK_MAX_SIZE 32 +struct gator_kmod_stack { unsigned long addresses[GATOR_KMOD_STACK_MAX_SIZE]; }; +static DEFINE_PER_CPU(struct gator_kmod_stack, gator_kmod_stack); + +static void report_trace(unsigned int cpu, struct stack_trace * trace) +{ + unsigned int cookie = NO_COOKIE; + unsigned int index; + + for (index = 0; index < trace->nr_entries; ++index) { + unsigned long addr = trace->entries[index]; +#if defined(MODULE) + struct module * mod = __module_address(addr); + if (mod) { + cookie = get_cookie(cpu, current, mod->name, false); + addr = addr - +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0) + (unsigned long) mod->module_core; +#else + (unsigned long) mod->core_layout.base; +#endif + } + else { + cookie = NO_COOKIE; + } +#endif + marshal_backtrace(addr & ~1, cookie, 1); + } +} + +static void kernel_backtrace(int cpu, struct pt_regs *const regs) +{ + struct stack_trace trace; + trace.skip = 0; + trace.nr_entries = 0; + trace.entries = per_cpu(gator_kmod_stack, cpu).addresses; +#ifdef GATOR_KERNEL_STACK_UNWINDING + trace.max_entries = gator_backtrace_depth; +#else + trace.max_entries = (kernel_stack_unwinding ? gator_backtrace_depth : 1); +#endif + if (trace.max_entries < 1) { + trace.max_entries = 1; + } + else if (trace.max_entries > GATOR_KMOD_STACK_MAX_SIZE) { + trace.max_entries = GATOR_KMOD_STACK_MAX_SIZE; + } + + save_stack_trace(&trace); + + report_trace(cpu, &trace); +} + + +#endif + +#else /* GATOR_KERNEL_UNWINDING */ + +/* -------------------------- NO KERNEL UNWINDING -------------------------- */ + +static void kernel_backtrace(int cpu, struct pt_regs *const regs) +{ + marshal_backtrace(PC_REG & ~1, NO_COOKIE, 1); +} + +#endif /* GATOR_KERNEL_UNWINDING */ + + +/* -------------------------- USER SPACE UNWINDING -------------------------- */ + +static void gator_add_trace(int cpu, unsigned long address) +{ + off_t offset = 0; + unsigned long cookie = get_address_cookie(cpu, current, address & ~1, &offset); + + if (cookie == NO_COOKIE || cookie == UNRESOLVED_COOKIE) + offset = address; + + marshal_backtrace(offset & ~1, cookie, 0); +} + +#if GATOR_USER_UNWINDING + +/* + * EABI backtrace stores {fp,lr} on the stack. + */ +struct stack_frame_eabi { + union { + struct { + unsigned long fp; + /* May be the fp in the case of a leaf function or clang */ + unsigned long lr; + /* If lr is really the fp, lr2 is the corresponding lr */ + unsigned long lr2; + }; + /* Used to read 32 bit fp/lr from a 64 bit kernel */ + struct { + u32 fp_32; + /* same as lr above */ + u32 lr_32; + /* same as lr2 above */ + u32 lr2_32; + }; + }; +}; + +static void arm_backtrace_eabi(int cpu, struct pt_regs *const regs, unsigned int depth) +{ + struct stack_frame_eabi *curr; + struct stack_frame_eabi bufcurr; +#if defined(__arm__) + const bool is_compat = false; + unsigned long fp = regs->ARM_fp; + unsigned long sp = regs->ARM_sp; + unsigned long lr = regs->ARM_lr; + const int gcc_frame_offset = sizeof(unsigned long); +#else + /* Is userspace aarch32 (32 bit) */ + const bool is_compat = compat_user_mode(regs); + unsigned long fp = (is_compat ? regs->regs[11] : regs->regs[29]); + unsigned long sp = (is_compat ? regs->compat_sp : regs->sp); + unsigned long lr = (is_compat ? regs->compat_lr : regs->regs[30]); + const int gcc_frame_offset = (is_compat ? sizeof(u32) : 0); +#endif + /* clang frame offset is always zero */ + int is_user_mode = user_mode(regs); + + /* pc (current function) has already been added */ + + if (!is_user_mode) + return; + + /* Add the lr (parent function), entry preamble may not have + * executed + */ + gator_add_trace(cpu, lr); + + /* check fp is valid */ + if (fp == 0 || fp < sp) + return; + + /* Get the current stack frame */ + curr = (struct stack_frame_eabi *)(fp - gcc_frame_offset); + if ((unsigned long)curr & 3) + return; + + while (depth-- && curr) { + if (!access_ok(VERIFY_READ, curr, sizeof(struct stack_frame_eabi)) || + __copy_from_user_inatomic(&bufcurr, curr, sizeof(struct stack_frame_eabi))) { + return; + } + + fp = (is_compat ? bufcurr.fp_32 : bufcurr.fp); + lr = (is_compat ? bufcurr.lr_32 : bufcurr.lr); + +#define calc_next(reg) ((reg) - gcc_frame_offset) + /* Returns true if reg is a valid fp */ +#define validate_next(reg, curr) \ + ((reg) != 0 && (calc_next(reg) & 3) == 0 && (unsigned long)(curr) < calc_next(reg)) + + /* Try lr from the stack as the fp because gcc leaf functions do + * not push lr. If gcc_frame_offset is non-zero, the lr will also + * be the clang fp. This assumes code is at a lower address than + * the stack + */ + if (validate_next(lr, curr)) { + fp = lr; + lr = (is_compat ? bufcurr.lr2_32 : bufcurr.lr2); + } + + gator_add_trace(cpu, lr); + + if (!validate_next(fp, curr)) + return; + + /* Move to the next stack frame */ + curr = (struct stack_frame_eabi *)calc_next(fp); + } +} + +#else /* GATOR_USER_UNWINDING */ + +static void arm_backtrace_eabi(int cpu, struct pt_regs *const regs, unsigned int depth) +{ + /* NO OP */ +} + +#endif /* GATOR_USER_UNWINDING */ + +/* -------------------------- STACK SAMPLING -------------------------- */ + +static void gator_add_sample(int cpu, struct pt_regs *const regs, u64 time) +{ + bool in_kernel; + unsigned long exec_cookie; + + if (!regs) + return; + + in_kernel = !user_mode(regs); + exec_cookie = get_exec_cookie(cpu, current); + + if (!marshal_backtrace_header(exec_cookie, current->tgid, current->pid, time)) + return; + + if (in_kernel) { + kernel_backtrace(cpu, regs); + } else { + /* Cookie+PC */ + gator_add_trace(cpu, PC_REG); + + /* Backtrace */ + if (gator_backtrace_depth) + arm_backtrace_eabi(cpu, regs, gator_backtrace_depth); + } + + marshal_backtrace_footer(time); +} diff --git a/drivers/gator/gator_buffer.c b/drivers/gator/gator_buffer.c new file mode 100644 index 000000000000..416ae55e52e0 --- /dev/null +++ b/drivers/gator/gator_buffer.c @@ -0,0 +1,171 @@ +/** + * Copyright (C) Arm Limited 2010-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +static void marshal_frame(int cpu, int buftype) +{ + int frame; + bool write_cpu; + + if (!per_cpu(gator_buffer, cpu)[buftype]) + return; + + switch (buftype) { + case SUMMARY_BUF: + write_cpu = false; + frame = FRAME_SUMMARY; + break; + case BACKTRACE_BUF: + write_cpu = true; + frame = FRAME_BACKTRACE; + break; + case NAME_BUF: + write_cpu = true; + frame = FRAME_NAME; + break; + case COUNTER_BUF: + write_cpu = false; + frame = FRAME_COUNTER; + break; + case BLOCK_COUNTER_BUF: + write_cpu = true; + frame = FRAME_BLOCK_COUNTER; + break; + case ANNOTATE_BUF: + write_cpu = false; + frame = FRAME_ANNOTATE; + break; + case SCHED_TRACE_BUF: + write_cpu = true; + frame = FRAME_SCHED_TRACE; + break; + case IDLE_BUF: + write_cpu = false; + frame = FRAME_IDLE; + break; + case ACTIVITY_BUF: + write_cpu = false; + frame = FRAME_ACTIVITY; + break; + default: + write_cpu = false; + frame = -1; + break; + } + + /* add response type */ + if (gator_response_type > 0) + gator_buffer_write_packed_int(cpu, buftype, gator_response_type); + + /* leave space for 4-byte unpacked length */ + per_cpu(gator_buffer_write, cpu)[buftype] = (per_cpu(gator_buffer_write, cpu)[buftype] + sizeof(s32)) & gator_buffer_mask[buftype]; + + /* add frame type and core number */ + gator_buffer_write_packed_int(cpu, buftype, frame); + if (write_cpu) + gator_buffer_write_packed_int(cpu, buftype, cpu); +} + +static int buffer_bytes_available(int cpu, int buftype) +{ + int remaining, filled; + + filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_read, cpu)[buftype]; + if (filled < 0) + filled += gator_buffer_size[buftype]; + + remaining = gator_buffer_size[buftype] - filled; + + if (per_cpu(buffer_space_available, cpu)[buftype]) + /* Give some extra room; also allows space to insert the overflow error packet */ + remaining -= 200; + else + /* Hysteresis, prevents multiple overflow messages */ + remaining -= 2000; + + return remaining; +} + +static bool buffer_check_space(int cpu, int buftype, int bytes) +{ + int remaining = buffer_bytes_available(cpu, buftype); + + if (remaining < bytes) + per_cpu(buffer_space_available, cpu)[buftype] = false; + else + per_cpu(buffer_space_available, cpu)[buftype] = true; + + return per_cpu(buffer_space_available, cpu)[buftype]; +} + +static int contiguous_space_available(int cpu, int buftype) +{ + int remaining = buffer_bytes_available(cpu, buftype); + int contiguous = gator_buffer_size[buftype] - per_cpu(gator_buffer_write, cpu)[buftype]; + + if (remaining < contiguous) + return remaining; + return contiguous; +} + +static void gator_commit_buffer(int cpu, int buftype, u64 time) +{ + int type_length, commit, length, byte; + unsigned long flags; + + if (!per_cpu(gator_buffer, cpu)[buftype]) + return; + + /* post-populate the length, which does not include the response type length nor the length itself, i.e. only the length of the payload */ + local_irq_save(flags); + type_length = gator_response_type ? 1 : 0; + commit = per_cpu(gator_buffer_commit, cpu)[buftype]; + length = per_cpu(gator_buffer_write, cpu)[buftype] - commit; + if (length < 0) + length += gator_buffer_size[buftype]; + length = length - type_length - sizeof(s32); + + if (length <= FRAME_HEADER_SIZE) { + /* Nothing to write, only the frame header is present */ + local_irq_restore(flags); + return; + } + + for (byte = 0; byte < sizeof(s32); byte++) + per_cpu(gator_buffer, cpu)[buftype][(commit + type_length + byte) & gator_buffer_mask[buftype]] = (length >> byte * 8) & 0xFF; + + per_cpu(gator_buffer_commit, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype]; + + if (gator_live_rate > 0) { + while (time > per_cpu(gator_buffer_commit_time, cpu)) + per_cpu(gator_buffer_commit_time, cpu) += gator_live_rate; + } + + marshal_frame(cpu, buftype); + local_irq_restore(flags); + + /* had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater */ + if (per_cpu(in_scheduler_context, cpu)) { +#ifndef CONFIG_PREEMPT_RT_FULL + /* mod_timer can not be used in interrupt context in RT-Preempt full */ + mod_timer(&gator_buffer_wake_up_timer, jiffies + 1); +#endif + } else { + up(&gator_buffer_wake_sem); + } +} + +static void buffer_check(int cpu, int buftype, u64 time) +{ + int filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_commit, cpu)[buftype]; + + if (filled < 0) + filled += gator_buffer_size[buftype]; + if (filled >= ((gator_buffer_size[buftype] * 3) / 4)) + gator_commit_buffer(cpu, buftype, time); +} diff --git a/drivers/gator/gator_buffer_write.c b/drivers/gator/gator_buffer_write.c new file mode 100644 index 000000000000..e6681882692f --- /dev/null +++ b/drivers/gator/gator_buffer_write.c @@ -0,0 +1,83 @@ +/** + * Copyright (C) Arm Limited 2010-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +static void gator_buffer_write_packed_int(int cpu, int buftype, int x) +{ + uint32_t write = per_cpu(gator_buffer_write, cpu)[buftype]; + uint32_t mask = gator_buffer_mask[buftype]; + char *buffer = per_cpu(gator_buffer, cpu)[buftype]; + int packedBytes = 0; + int more = true; + + while (more) { + /* low order 7 bits of x */ + char b = x & 0x7f; + + x >>= 7; + + if ((x == 0 && (b & 0x40) == 0) || (x == -1 && (b & 0x40) != 0)) + more = false; + else + b |= 0x80; + + buffer[(write + packedBytes) & mask] = b; + packedBytes++; + } + + per_cpu(gator_buffer_write, cpu)[buftype] = (write + packedBytes) & mask; +} + +static void gator_buffer_write_packed_int64(int cpu, int buftype, long long x) +{ + uint32_t write = per_cpu(gator_buffer_write, cpu)[buftype]; + uint32_t mask = gator_buffer_mask[buftype]; + char *buffer = per_cpu(gator_buffer, cpu)[buftype]; + int packedBytes = 0; + int more = true; + + while (more) { + /* low order 7 bits of x */ + char b = x & 0x7f; + + x >>= 7; + + if ((x == 0 && (b & 0x40) == 0) || (x == -1 && (b & 0x40) != 0)) + more = false; + else + b |= 0x80; + + buffer[(write + packedBytes) & mask] = b; + packedBytes++; + } + + per_cpu(gator_buffer_write, cpu)[buftype] = (write + packedBytes) & mask; +} + +static void gator_buffer_write_bytes(int cpu, int buftype, const char *x, int len) +{ + int i; + u32 write = per_cpu(gator_buffer_write, cpu)[buftype]; + u32 mask = gator_buffer_mask[buftype]; + char *buffer = per_cpu(gator_buffer, cpu)[buftype]; + + for (i = 0; i < len; i++) { + buffer[write] = x[i]; + write = (write + 1) & mask; + } + + per_cpu(gator_buffer_write, cpu)[buftype] = write; +} + +static void gator_buffer_write_string(int cpu, int buftype, const char *x) +{ + int len = strlen(x); + + gator_buffer_write_packed_int(cpu, buftype, len); + gator_buffer_write_bytes(cpu, buftype, x, len); +} diff --git a/drivers/gator/gator_cookies.c b/drivers/gator/gator_cookies.c new file mode 100644 index 000000000000..1a5076c99cb9 --- /dev/null +++ b/drivers/gator/gator_cookies.c @@ -0,0 +1,550 @@ +/** + * Copyright (C) Arm Limited 2010-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) +# include +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) +/* Kernel version 4.10.0 adds locked argument + (See https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/commit/include/linux/mm.h?id=5b56d49fc31dbb0487e14ead790fc81ca9fb2c99) */ +# define get_user_pages_remote(tsk,mm,start,nr_pages,write,force,pages,vmas) get_user_pages_remote(tsk,mm,start,nr_pages,((write) ? FOLL_WRITE : 0) | ((force) ? FOLL_FORCE : 0),pages,vmas,NULL) +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0) +/* Kernel version 4.9.0 removes write and force arguments from get_user_pages_remote and replaces with gup_flags instead + (See https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/commit/include/linux/mm.h?id=9beae1ea89305a9667ceaab6d0bf46a045ad71e7) */ +# define get_user_pages_remote(tsk,mm,start,nr_pages,write,force,pages,vmas) get_user_pages_remote(tsk,mm,start,nr_pages,((write) ? FOLL_WRITE : 0) | ((force) ? FOLL_FORCE : 0),pages,vmas) +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) +/* Kernel version 4.6.0 removes get_user_pages macro. We should use get_user_pages_remote anyway. + (See https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/commit/include/linux/mm.h?id=c12d2da56d0e07d230968ee2305aaa86b93a6832) */ +# define get_user_pages_remote(tsk,mm,start,nr_pages,write,force,pages,vmas) get_user_pages(tsk,mm,start,nr_pages,write,force,pages,vmas) +#endif + +struct mount { + struct mount *mnt_parent; + struct dentry *mnt_mountpoint; + struct vfsmount mnt; +}; + +static inline struct mount *real_mount(struct vfsmount *mnt) +{ + return container_of(mnt, struct mount, mnt); +} + +#define GET_MNT_ROOT(mount) ((mount)->mnt.mnt_root) + +/* must be power of 2 */ +#define COOKIEMAP_ENTRIES 1024 +/* must be a power of 2 - 512/4 = 128 entries */ +#define TRANSLATE_BUFFER_SIZE 512 +#define TRANSLATE_TEXT_SIZE 256 +#define MAX_COLLISIONS 2 + +static uint32_t *gator_crc32_table; +static unsigned int translate_buffer_mask; + +struct cookie_args { + struct task_struct *task; + const char *text; +}; + +static DEFINE_PER_CPU(char *, translate_text); +static DEFINE_PER_CPU(char *, scratch); +static DEFINE_PER_CPU(uint32_t, cookie_next_key); +static DEFINE_PER_CPU(uint64_t *, cookie_keys); +static DEFINE_PER_CPU(uint32_t *, cookie_values); +static DEFINE_PER_CPU(int, translate_buffer_read); +static DEFINE_PER_CPU(int, translate_buffer_write); +static DEFINE_PER_CPU(struct cookie_args *, translate_buffer); + +static uint32_t get_cookie(int cpu, struct task_struct *task, const char *text, bool from_wq); +static void wq_cookie_handler(struct work_struct *unused); +static DECLARE_WORK(cookie_work, wq_cookie_handler); +static struct timer_list app_process_wake_up_timer; +static DECLARE_TIMER_HANDLER(app_process_wake_up_handler); + +static uint32_t cookiemap_code(uint64_t value64) +{ + uint32_t value = (uint32_t)((value64 >> 32) + value64); + uint32_t cookiecode = (value >> 24) & 0xff; + + cookiecode = cookiecode * 31 + ((value >> 16) & 0xff); + cookiecode = cookiecode * 31 + ((value >> 8) & 0xff); + cookiecode = cookiecode * 31 + ((value >> 0) & 0xff); + cookiecode &= (COOKIEMAP_ENTRIES - 1); + return cookiecode * MAX_COLLISIONS; +} + +static uint32_t gator_chksum_crc32(const char *data) +{ + register unsigned long crc; + const unsigned char *block = data; + int i, length = strlen(data); + + crc = 0xFFFFFFFF; + for (i = 0; i < length; i++) + crc = ((crc >> 8) & 0x00FFFFFF) ^ gator_crc32_table[(crc ^ *block++) & 0xFF]; + + return (crc ^ 0xFFFFFFFF); +} + +/* + * Exists + * Pre: [0][1][v][3]..[n-1] + * Post: [v][0][1][3]..[n-1] + */ +static uint32_t cookiemap_exists(uint64_t key) +{ + unsigned long x, flags, retval = 0; + int cpu = get_physical_cpu(); + uint32_t cookiecode = cookiemap_code(key); + uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]); + uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]); + + /* Can be called from interrupt handler or from work queue */ + local_irq_save(flags); + for (x = 0; x < MAX_COLLISIONS; x++) { + if (keys[x] == key) { + uint32_t value = values[x]; + + for (; x > 0; x--) { + keys[x] = keys[x - 1]; + values[x] = values[x - 1]; + } + keys[0] = key; + values[0] = value; + retval = value; + break; + } + } + local_irq_restore(flags); + + return retval; +} + +/* + * Add + * Pre: [0][1][2][3]..[n-1] + * Post: [v][0][1][2]..[n-2] + */ +static void cookiemap_add(uint64_t key, uint32_t value) +{ + int cpu = get_physical_cpu(); + int cookiecode = cookiemap_code(key); + uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]); + uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]); + int x; + + for (x = MAX_COLLISIONS - 1; x > 0; x--) { + keys[x] = keys[x - 1]; + values[x] = values[x - 1]; + } + keys[0] = key; + values[0] = value; +} + +#ifndef CONFIG_PREEMPT_RT_FULL +static void translate_buffer_write_args(int cpu, struct task_struct *task, const char *text) +{ + unsigned long flags; + int write; + int next_write; + struct cookie_args *args; + + local_irq_save(flags); + + write = per_cpu(translate_buffer_write, cpu); + next_write = (write + 1) & translate_buffer_mask; + + /* At least one entry must always remain available as when read == write, the queue is empty not full */ + if (next_write != per_cpu(translate_buffer_read, cpu)) { + args = &per_cpu(translate_buffer, cpu)[write]; + args->task = task; + args->text = text; + get_task_struct(task); + per_cpu(translate_buffer_write, cpu) = next_write; + } + + local_irq_restore(flags); +} +#endif + +static void translate_buffer_read_args(int cpu, struct cookie_args *args) +{ + unsigned long flags; + int read; + + local_irq_save(flags); + + read = per_cpu(translate_buffer_read, cpu); + *args = per_cpu(translate_buffer, cpu)[read]; + per_cpu(translate_buffer_read, cpu) = (read + 1) & translate_buffer_mask; + + local_irq_restore(flags); +} + +static void wq_cookie_handler(struct work_struct *unused) +{ + struct cookie_args args; + int cpu = get_physical_cpu(), cookie; + + mutex_lock(&start_mutex); + + if (gator_started != 0) { + while (per_cpu(translate_buffer_read, cpu) != per_cpu(translate_buffer_write, cpu)) { + translate_buffer_read_args(cpu, &args); + cookie = get_cookie(cpu, args.task, args.text, true); + marshal_link(cookie, args.task->tgid, args.task->pid); + put_task_struct(args.task); + } + } + + mutex_unlock(&start_mutex); +} + +static DECLARE_TIMER_HANDLER(app_process_wake_up_handler) +{ + /* had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater */ + schedule_work(&cookie_work); +} + +/* Retrieve full name from proc/pid/cmdline for java processes on Android */ +static int translate_app_process(const char **text, int cpu, struct task_struct *task, bool from_wq) +{ + void *maddr; + unsigned int len; + unsigned long addr; + struct mm_struct *mm; + struct page *page = NULL; + struct vm_area_struct *page_vma; + int bytes, offset, retval = 0; + char *buf = per_cpu(translate_text, cpu); + +#ifndef CONFIG_PREEMPT_RT_FULL + /* Push work into a work queue if in atomic context as the kernel + * functions below might sleep. Rely on the in_interrupt variable + * rather than in_irq() or in_interrupt() kernel functions, as the + * value of these functions seems inconsistent during a context + * switch between android/linux versions + */ + if (!from_wq) { + /* Check if already in buffer */ + int pos = per_cpu(translate_buffer_read, cpu); + + while (pos != per_cpu(translate_buffer_write, cpu)) { + if (per_cpu(translate_buffer, cpu)[pos].task == task) + goto out; + pos = (pos + 1) & translate_buffer_mask; + } + + translate_buffer_write_args(cpu, task, *text); + + /* Not safe to call in RT-Preempt full in schedule switch context */ + mod_timer(&app_process_wake_up_timer, jiffies + 1); + goto out; + } +#endif + + mm = get_task_mm(task); + if (!mm) + goto out; + if (!mm->arg_end) + goto outmm; + addr = mm->arg_start; + len = mm->arg_end - mm->arg_start; + + if (len > TRANSLATE_TEXT_SIZE) + len = TRANSLATE_TEXT_SIZE; + + down_read(&mm->mmap_sem); + while (len) { + if (get_user_pages_remote(task, mm, addr, 1, 0, 1, &page, &page_vma) <= 0) + goto outsem; + + maddr = kmap(page); + offset = addr & (PAGE_SIZE - 1); + bytes = len; + if (bytes > PAGE_SIZE - offset) + bytes = PAGE_SIZE - offset; + + copy_from_user_page(page_vma, page, addr, buf, maddr + offset, bytes); + + /* release page allocated by get_user_pages_remote() */ + kunmap(page); + page_cache_release(page); + + len -= bytes; + buf += bytes; + addr += bytes; + + *text = per_cpu(translate_text, cpu); + retval = 1; + } + + /* On app_process startup, /proc/pid/cmdline is initially "zygote" then "" but changes after an initial startup period */ + if (strcmp(*text, "zygote") == 0 || strcmp(*text, "") == 0) + retval = 0; + +outsem: + up_read(&mm->mmap_sem); +outmm: + mmput(mm); +out: + return retval; +} + +static const char APP_PROCESS[] = "app_process"; + +static uint32_t get_cookie(int cpu, struct task_struct *task, const char *text, bool from_wq) +{ + unsigned long flags, cookie; + uint64_t key; + + key = gator_chksum_crc32(text); + key = (key << 32) | (uint32_t)task->tgid; + + cookie = cookiemap_exists(key); + if (cookie) + return cookie; + + /* On 64-bit android app_process can be app_process32 or app_process64 */ + if (strstr(text, APP_PROCESS) != NULL) { + if (!translate_app_process(&text, cpu, task, from_wq)) + return UNRESOLVED_COOKIE; + } + + /* Can be called from interrupt handler or from work queue or from scheduler trace */ + local_irq_save(flags); + + cookie = UNRESOLVED_COOKIE; + if (marshal_cookie_header(text)) { + cookie = per_cpu(cookie_next_key, cpu) += nr_cpu_ids; + cookiemap_add(key, cookie); + marshal_cookie(cookie, text); + } + + local_irq_restore(flags); + + return cookie; +} + +/* Can't call d_path in interrupt context so create something similar */ +static const char *gator_d_path(const struct path *path, char *buf, int buflen) +{ + struct dentry *dentry = path->dentry; + struct mount *mount = real_mount(path->mnt); + int pos = buflen - 1; + int len; + + buf[pos] = '\0'; + + for (;;) { + if (dentry == NULL) { + pr_err("gator: dentry is null!\n"); + break; + } + if (dentry->d_name.name[0] == '\0') { + pr_err("gator: path is empty string\n"); + break; + } + if (dentry->d_name.name[0] == '/' && dentry->d_name.name[1] == '\0') { + /* Normal operation */ + /* pr_err("gator: path is /\n"); */ + break; + } + + len = strlen(dentry->d_name.name); + if (pos < len) { + pr_err("gator: path is too long\n"); + break; + } + pos -= len; + memcpy(buf + pos, dentry->d_name.name, len); + + if (pos == 0) { + pr_err("gator: no room for slash\n"); + /* Fall back to name only */ + return path->dentry->d_name.name; + } + --pos; + buf[pos] = '/'; + + if (dentry->d_parent == GET_MNT_ROOT(mount)) { + /* pr_err("gator: filesystem is complete, moving to next '%s'\n", buf + pos); */ + dentry = mount->mnt_mountpoint; + mount = mount->mnt_parent; + continue; + } + if (dentry == dentry->d_parent) { + /* Normal operation, at least for ashmem */ + /* pr_err("gator: parent is self\n"); */ + break; + } + dentry = dentry->d_parent; + } + + if (pos < 0) { + pr_err("gator: pos is somenow negative\n"); + /* Fall back to name only */ + return path->dentry->d_name.name; + } + + return buf + pos; +} + +#define d_path gator_d_path + +static int get_exec_cookie(int cpu, struct task_struct *task) +{ + struct mm_struct *mm = task->mm; + const char *text; + + /* kernel threads have no address space */ + if (!mm) + return NO_COOKIE; + + if (task && task->mm && task->mm->exe_file) { + text = d_path(&task->mm->exe_file->f_path, per_cpu(scratch, get_physical_cpu()), PAGE_SIZE); + return get_cookie(cpu, task, text, false); + } + + return UNRESOLVED_COOKIE; +} + +static unsigned long get_address_cookie(int cpu, struct task_struct *task, unsigned long addr, off_t *offset) +{ + unsigned long cookie = NO_COOKIE; + struct mm_struct *mm = task->mm; + struct vm_area_struct *vma; + const char *text; + + if (!mm) + return cookie; + + for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) { + if (addr < vma->vm_start || addr >= vma->vm_end) + continue; + + if (vma->vm_file) { + text = d_path(&vma->vm_file->f_path, per_cpu(scratch, get_physical_cpu()), PAGE_SIZE); + cookie = get_cookie(cpu, task, text, false); + *offset = (vma->vm_pgoff << PAGE_SHIFT) + addr - vma->vm_start; + } else { + /* must be an anonymous map */ + *offset = addr; + } + + break; + } + + if (!vma) + cookie = UNRESOLVED_COOKIE; + + return cookie; +} + +static int cookies_initialize(void) +{ + uint32_t crc, poly; + int i, j, cpu, size, err = 0; + + translate_buffer_mask = TRANSLATE_BUFFER_SIZE / sizeof(per_cpu(translate_buffer, 0)[0]) - 1; + + for_each_present_cpu(cpu) { + per_cpu(cookie_next_key, cpu) = nr_cpu_ids + cpu; + + size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint64_t); + per_cpu(cookie_keys, cpu) = kmalloc(size, GFP_KERNEL); + if (!per_cpu(cookie_keys, cpu)) { + err = -ENOMEM; + goto cookie_setup_error; + } + memset(per_cpu(cookie_keys, cpu), 0, size); + + size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint32_t); + per_cpu(cookie_values, cpu) = kmalloc(size, GFP_KERNEL); + if (!per_cpu(cookie_values, cpu)) { + err = -ENOMEM; + goto cookie_setup_error; + } + memset(per_cpu(cookie_values, cpu), 0, size); + + per_cpu(translate_buffer, cpu) = kmalloc(TRANSLATE_BUFFER_SIZE, GFP_KERNEL); + if (!per_cpu(translate_buffer, cpu)) { + err = -ENOMEM; + goto cookie_setup_error; + } + + per_cpu(translate_buffer_write, cpu) = 0; + per_cpu(translate_buffer_read, cpu) = 0; + + per_cpu(translate_text, cpu) = kmalloc(TRANSLATE_TEXT_SIZE, GFP_KERNEL); + if (!per_cpu(translate_text, cpu)) { + err = -ENOMEM; + goto cookie_setup_error; + } + + per_cpu(scratch, cpu) = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!per_cpu(scratch, cpu)) { + err = -ENOMEM; + goto cookie_setup_error; + } + } + + /* build CRC32 table */ + poly = 0x04c11db7; + gator_crc32_table = kmalloc(256 * sizeof(*gator_crc32_table), GFP_KERNEL); + if (!gator_crc32_table) { + err = -ENOMEM; + goto cookie_setup_error; + } + for (i = 0; i < 256; i++) { + crc = i; + for (j = 8; j > 0; j--) { + if (crc & 1) + crc = (crc >> 1) ^ poly; + else + crc >>= 1; + } + gator_crc32_table[i] = crc; + } + + setup_deferrable_timer_on_stack(&app_process_wake_up_timer, app_process_wake_up_handler, 0); + +cookie_setup_error: + return err; +} + +static void cookies_release(void) +{ + int cpu; + + for_each_present_cpu(cpu) { + kfree(per_cpu(cookie_keys, cpu)); + per_cpu(cookie_keys, cpu) = NULL; + + kfree(per_cpu(cookie_values, cpu)); + per_cpu(cookie_values, cpu) = NULL; + + kfree(per_cpu(translate_buffer, cpu)); + per_cpu(translate_buffer, cpu) = NULL; + per_cpu(translate_buffer_read, cpu) = 0; + per_cpu(translate_buffer_write, cpu) = 0; + + kfree(per_cpu(translate_text, cpu)); + per_cpu(translate_text, cpu) = NULL; + + kfree(per_cpu(scratch, cpu)); + per_cpu(scratch, cpu) = NULL; + } + + del_timer_sync(&app_process_wake_up_timer); + kfree(gator_crc32_table); + gator_crc32_table = NULL; +} diff --git a/drivers/gator/gator_events_block.c b/drivers/gator/gator_events_block.c new file mode 100644 index 000000000000..32d2a3d53214 --- /dev/null +++ b/drivers/gator/gator_events_block.c @@ -0,0 +1,162 @@ +/** + * Copyright (C) Arm Limited 2010-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#include "gator.h" +#include + +#define BLOCK_RQ_WR 0 +#define BLOCK_RQ_RD 1 + +#define BLOCK_TOTAL (BLOCK_RQ_RD+1) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0) +#define GATOR_REQ_IS_OP_WRITE(rq) (rq->cmd_flags & REQ_WRITE) +#else +#define GATOR_REQ_IS_OP_WRITE(rq) (req_op(rq) & REQ_OP_WRITE) +#endif + +static ulong block_rq_wr_enabled; +static ulong block_rq_rd_enabled; +static ulong block_rq_wr_key; +static ulong block_rq_rd_key; +static atomic_t blockCnt[BLOCK_TOTAL]; +static int blockGet[BLOCK_TOTAL * 4]; + +/* Tracepoint changed in 3.15 backported to older kernels. The Makefile tries to autodetect the correct value, but if it fails change the #if below */ +#if OLD_BLOCK_RQ_COMPLETE +GATOR_DEFINE_PROBE(block_rq_complete, TP_PROTO(struct request_queue *q, struct request *rq)) +#else +GATOR_DEFINE_PROBE(block_rq_complete, TP_PROTO(struct request_queue *q, struct request *rq, unsigned int nr_bytes)) +#endif +{ + int write; + unsigned int size; + + if (!rq) + return; + + write = GATOR_REQ_IS_OP_WRITE(rq); + +#if OLD_BLOCK_RQ_COMPLETE + size = rq->resid_len; +#else + size = nr_bytes; +#endif + + if (!size) + return; + + if (write) { + if (block_rq_wr_enabled) + atomic_add(size, &blockCnt[BLOCK_RQ_WR]); + } else { + if (block_rq_rd_enabled) + atomic_add(size, &blockCnt[BLOCK_RQ_RD]); + } +} + +static int gator_events_block_create_files(struct super_block *sb, struct dentry *root) +{ + struct dentry *dir; + + /* block_complete_wr */ + dir = gatorfs_mkdir(sb, root, "Linux_block_rq_wr"); + if (!dir) + return -1; + gatorfs_create_ulong(sb, dir, "enabled", &block_rq_wr_enabled); + gatorfs_create_ro_ulong(sb, dir, "key", &block_rq_wr_key); + + /* block_complete_rd */ + dir = gatorfs_mkdir(sb, root, "Linux_block_rq_rd"); + if (!dir) + return -1; + gatorfs_create_ulong(sb, dir, "enabled", &block_rq_rd_enabled); + gatorfs_create_ro_ulong(sb, dir, "key", &block_rq_rd_key); + + return 0; +} + +static int gator_events_block_start(void) +{ + /* register tracepoints */ + if (block_rq_wr_enabled || block_rq_rd_enabled) + if (GATOR_REGISTER_TRACE(block_rq_complete)) + goto fail_block_rq_exit; + pr_debug("gator: registered block event tracepoints\n"); + + return 0; + + /* unregister tracepoints on error */ +fail_block_rq_exit: + pr_err("gator: block event tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n"); + + return -1; +} + +static void gator_events_block_stop(void) +{ + if (block_rq_wr_enabled || block_rq_rd_enabled) + GATOR_UNREGISTER_TRACE(block_rq_complete); + pr_debug("gator: unregistered block event tracepoints\n"); + + block_rq_wr_enabled = 0; + block_rq_rd_enabled = 0; +} + +static int gator_events_block_read(int **buffer, bool sched_switch) +{ + int len, value, data = 0; + + if (!on_primary_core()) + return 0; + + len = 0; + if (block_rq_wr_enabled && (value = atomic_read(&blockCnt[BLOCK_RQ_WR])) > 0) { + atomic_sub(value, &blockCnt[BLOCK_RQ_WR]); + blockGet[len++] = block_rq_wr_key; + /* Indicates to Streamline that value bytes were written now, not since the last message */ + blockGet[len++] = 0; + blockGet[len++] = block_rq_wr_key; + blockGet[len++] = value; + data += value; + } + if (block_rq_rd_enabled && (value = atomic_read(&blockCnt[BLOCK_RQ_RD])) > 0) { + atomic_sub(value, &blockCnt[BLOCK_RQ_RD]); + blockGet[len++] = block_rq_rd_key; + /* Indicates to Streamline that value bytes were read now, not since the last message */ + blockGet[len++] = 0; + blockGet[len++] = block_rq_rd_key; + blockGet[len++] = value; + data += value; + } + + if (buffer) + *buffer = blockGet; + + return len; +} + +static struct gator_interface gator_events_block_interface = { + .name = "block", + .create_files = gator_events_block_create_files, + .start = gator_events_block_start, + .stop = gator_events_block_stop, + .read = gator_events_block_read, +}; + +int gator_events_block_init(void) +{ + block_rq_wr_enabled = 0; + block_rq_rd_enabled = 0; + + block_rq_wr_key = gator_events_get_key(); + block_rq_rd_key = gator_events_get_key(); + + return gator_events_install(&gator_events_block_interface); +} diff --git a/drivers/gator/gator_events_irq.c b/drivers/gator/gator_events_irq.c new file mode 100644 index 000000000000..4c78607b218e --- /dev/null +++ b/drivers/gator/gator_events_irq.c @@ -0,0 +1,189 @@ +/** + * Copyright (C) Arm Limited 2010-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#include "gator.h" +#include + +#define HARDIRQ 0 +#define SOFTIRQ 1 +#define TOTALIRQ (SOFTIRQ+1) + +static ulong hardirq_enabled[GATOR_CLUSTER_COUNT]; +static bool hardirq_enabled_any; +static ulong softirq_enabled[GATOR_CLUSTER_COUNT]; +static bool softirq_enabled_any; +static ulong hardirq_key[GATOR_CLUSTER_COUNT]; +static ulong softirq_key[GATOR_CLUSTER_COUNT]; +static DEFINE_PER_CPU(atomic_t[TOTALIRQ], irqCnt); +static DEFINE_PER_CPU(int[TOTALIRQ * 2], irqGet); + +GATOR_DEFINE_PROBE(irq_handler_exit, + TP_PROTO(int irq, struct irqaction *action, int ret)) +{ + atomic_inc(&per_cpu(irqCnt, get_physical_cpu())[HARDIRQ]); +} + +GATOR_DEFINE_PROBE(softirq_exit, TP_PROTO(unsigned int vec_nr)) +{ + atomic_inc(&per_cpu(irqCnt, get_physical_cpu())[SOFTIRQ]); +} + +static int gator_events_irq_create_files(struct super_block *sb, struct dentry *root) +{ + struct dentry *dir; + int i; + char buf[40]; + + /* irq */ + for (i = 0; i < gator_cluster_count; i++) { + snprintf(buf, sizeof(buf), "%s_irq", gator_clusters[i]->pmnc_name); + dir = gatorfs_mkdir(sb, root, buf); + if (!dir) + return -1; + gatorfs_create_ulong(sb, dir, "enabled", &hardirq_enabled[i]); + gatorfs_create_ro_ulong(sb, dir, "key", &hardirq_key[i]); + } + + /* soft irq */ + for (i = 0; i < gator_cluster_count; i++) { + snprintf(buf, sizeof(buf), "%s_softirq", gator_clusters[i]->pmnc_name); + dir = gatorfs_mkdir(sb, root, buf); + if (!dir) + return -1; + gatorfs_create_ulong(sb, dir, "enabled", &softirq_enabled[i]); + gatorfs_create_ro_ulong(sb, dir, "key", &softirq_key[i]); + } + + return 0; +} + +static int gator_events_irq_online(int **buffer, bool migrate) +{ + int len = 0, cpu = get_physical_cpu(); + int cluster = gator_clusterids[cpu]; + + /* synchronization with the irq_exit functions is not necessary as the values are being reset */ + if (hardirq_enabled[cluster]) { + atomic_set(&per_cpu(irqCnt, cpu)[HARDIRQ], 0); + per_cpu(irqGet, cpu)[len++] = hardirq_key[cluster]; + per_cpu(irqGet, cpu)[len++] = 0; + } + + if (softirq_enabled[cluster]) { + atomic_set(&per_cpu(irqCnt, cpu)[SOFTIRQ], 0); + per_cpu(irqGet, cpu)[len++] = softirq_key[cluster]; + per_cpu(irqGet, cpu)[len++] = 0; + } + + if (buffer) + *buffer = per_cpu(irqGet, cpu); + + return len; +} + +static int gator_events_irq_start(void) +{ + int i; + + hardirq_enabled_any = false; + softirq_enabled_any = false; + for (i = 0; i < gator_cluster_count; i++) { + if (hardirq_enabled[i]) + hardirq_enabled_any = true; + if (softirq_enabled[i]) + softirq_enabled_any = true; + } + + /* register tracepoints */ + if (hardirq_enabled_any) + if (GATOR_REGISTER_TRACE(irq_handler_exit)) + goto fail_hardirq_exit; + if (softirq_enabled_any) + if (GATOR_REGISTER_TRACE(softirq_exit)) + goto fail_softirq_exit; + pr_debug("gator: registered irq tracepoints\n"); + + return 0; + + /* unregister tracepoints on error */ +fail_softirq_exit: + if (hardirq_enabled_any) + GATOR_UNREGISTER_TRACE(irq_handler_exit); +fail_hardirq_exit: + pr_err("gator: irq tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n"); + + return -1; +} + +static void gator_events_irq_stop(void) +{ + if (hardirq_enabled_any) + GATOR_UNREGISTER_TRACE(irq_handler_exit); + if (softirq_enabled_any) + GATOR_UNREGISTER_TRACE(softirq_exit); + pr_debug("gator: unregistered irq tracepoints\n"); + + hardirq_enabled_any = false; + memset(hardirq_enabled, 0, sizeof(hardirq_enabled)); + softirq_enabled_any = false; + memset(softirq_enabled, 0, sizeof(softirq_enabled)); +} + +static int gator_events_irq_read(int **buffer, bool sched_switch) +{ + int len, value; + int cpu = get_physical_cpu(); + int cluster = gator_clusterids[cpu]; + + len = 0; + if (hardirq_enabled[cluster]) { + value = atomic_read(&per_cpu(irqCnt, cpu)[HARDIRQ]); + atomic_sub(value, &per_cpu(irqCnt, cpu)[HARDIRQ]); + + per_cpu(irqGet, cpu)[len++] = hardirq_key[cluster]; + per_cpu(irqGet, cpu)[len++] = value; + } + + if (softirq_enabled[cluster]) { + value = atomic_read(&per_cpu(irqCnt, cpu)[SOFTIRQ]); + atomic_sub(value, &per_cpu(irqCnt, cpu)[SOFTIRQ]); + + per_cpu(irqGet, cpu)[len++] = softirq_key[cluster]; + per_cpu(irqGet, cpu)[len++] = value; + } + + if (buffer) + *buffer = per_cpu(irqGet, cpu); + + return len; +} + +static struct gator_interface gator_events_irq_interface = { + .name = "irq", + .create_files = gator_events_irq_create_files, + .online = gator_events_irq_online, + .start = gator_events_irq_start, + .stop = gator_events_irq_stop, + .read = gator_events_irq_read, +}; + +int gator_events_irq_init(void) +{ + int i; + + for (i = 0; i < gator_cluster_count; i++) { + hardirq_key[i] = gator_events_get_key(); + softirq_key[i] = gator_events_get_key(); + + hardirq_enabled[i] = 0; + softirq_enabled[i] = 0; + } + + return gator_events_install(&gator_events_irq_interface); +} diff --git a/drivers/gator/gator_events_l2c-310.c b/drivers/gator/gator_events_l2c-310.c new file mode 100644 index 000000000000..4a8c4b5c28f8 --- /dev/null +++ b/drivers/gator/gator_events_l2c-310.c @@ -0,0 +1,209 @@ +/** + * l2c310 (L2 Cache Controller) event counters for gator + * + * Copyright (C) Arm Limited 2010-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#if defined(CONFIG_OF) +#include +#include +#endif +#include + +#include "gator.h" + +#define L2C310_COUNTERS_NUM 2 + +static struct { + unsigned long enabled; + unsigned long event; + unsigned long key; +} l2c310_counters[L2C310_COUNTERS_NUM]; + +static int l2c310_buffer[L2C310_COUNTERS_NUM * 2]; + +static void __iomem *l2c310_base; + +static void gator_events_l2c310_reset_counters(void) +{ + u32 val = readl(l2c310_base + L2X0_EVENT_CNT_CTRL); + + val |= ((1 << L2C310_COUNTERS_NUM) - 1) << 1; + + writel(val, l2c310_base + L2X0_EVENT_CNT_CTRL); +} + +static int gator_events_l2c310_create_files(struct super_block *sb, + struct dentry *root) +{ + int i; + + for (i = 0; i < L2C310_COUNTERS_NUM; i++) { + char buf[16]; + struct dentry *dir; + + snprintf(buf, sizeof(buf), "l2c_310_cnt%d", i); + dir = gatorfs_mkdir(sb, root, buf); + if (WARN_ON(!dir)) + return -1; + gatorfs_create_ulong(sb, dir, "enabled", + &l2c310_counters[i].enabled); + gatorfs_create_ulong(sb, dir, "event", + &l2c310_counters[i].event); + gatorfs_create_ro_ulong(sb, dir, "key", + &l2c310_counters[i].key); + } + + return 0; +} + +static int gator_events_l2c310_start(void) +{ + static const unsigned long l2x0_event_cntx_cfg[L2C310_COUNTERS_NUM] = { + L2X0_EVENT_CNT0_CFG, + L2X0_EVENT_CNT1_CFG, + }; + int i; + + /* Counter event sources */ + for (i = 0; i < L2C310_COUNTERS_NUM; i++) + writel((l2c310_counters[i].event & 0xf) << 2, + l2c310_base + l2x0_event_cntx_cfg[i]); + + gator_events_l2c310_reset_counters(); + + /* Event counter enable */ + writel(1, l2c310_base + L2X0_EVENT_CNT_CTRL); + + return 0; +} + +static void gator_events_l2c310_stop(void) +{ + /* Event counter disable */ + writel(0, l2c310_base + L2X0_EVENT_CNT_CTRL); +} + +static int gator_events_l2c310_read(int **buffer, bool sched_switch) +{ + static const unsigned long l2x0_event_cntx_val[L2C310_COUNTERS_NUM] = { + L2X0_EVENT_CNT0_VAL, + L2X0_EVENT_CNT1_VAL, + }; + int i; + int len = 0; + + if (!on_primary_core()) + return 0; + + for (i = 0; i < L2C310_COUNTERS_NUM; i++) { + if (l2c310_counters[i].enabled) { + l2c310_buffer[len++] = l2c310_counters[i].key; + l2c310_buffer[len++] = readl(l2c310_base + + l2x0_event_cntx_val[i]); + } + } + + /* l2c310 counters are saturating, not wrapping in case of overflow */ + gator_events_l2c310_reset_counters(); + + if (buffer) + *buffer = l2c310_buffer; + + return len; +} + +static struct gator_interface gator_events_l2c310_interface = { + .name = "l2c-310", + .create_files = gator_events_l2c310_create_files, + .start = gator_events_l2c310_start, + .stop = gator_events_l2c310_stop, + .read = gator_events_l2c310_read, +}; + +#define L2C310_ADDR_PROBE (~0) + +MODULE_PARM_DESC(l2c310_addr, "L2C310 physical base address (0 to disable)"); +static unsigned long l2c310_addr = L2C310_ADDR_PROBE; +module_param(l2c310_addr, ulong, 0444); + +static void __iomem *gator_events_l2c310_probe(void) +{ + phys_addr_t variants[] = { +#if defined(CONFIG_ARCH_EXYNOS4) || defined(CONFIG_ARCH_S5PV310) + 0x10502000, +#endif +#if defined(CONFIG_ARCH_OMAP4) + 0x48242000, +#endif +#if defined(CONFIG_ARCH_TEGRA) + 0x50043000, +#endif +#if defined(CONFIG_ARCH_U8500) + 0xa0412000, +#endif +#if defined(CONFIG_ARCH_VEXPRESS) + 0x1e00a000, /* A9x4 core tile (HBI-0191) */ + 0x2c0f0000, /* New memory map tiles */ +#endif + }; + int i; + void __iomem *base; +#if defined(CONFIG_OF) + struct device_node *node = of_find_all_nodes(NULL); + + if (node) { + of_node_put(node); + + node = of_find_compatible_node(NULL, NULL, "arm,pl310-cache"); + base = of_iomap(node, 0); + of_node_put(node); + + return base; + } +#endif + + for (i = 0; i < ARRAY_SIZE(variants); i++) { + base = ioremap(variants[i], SZ_4K); + if (base) { + u32 cache_id = readl(base + L2X0_CACHE_ID); + + if ((cache_id & 0xff0003c0) == 0x410000c0) + return base; + + iounmap(base); + } + } + + return NULL; +} + +int gator_events_l2c310_init(void) +{ + int i; + + if (gator_cpuid() != CORTEX_A5 && gator_cpuid() != CORTEX_A9) + return -1; + + if (l2c310_addr == L2C310_ADDR_PROBE) + l2c310_base = gator_events_l2c310_probe(); + else if (l2c310_addr) + l2c310_base = ioremap(l2c310_addr, SZ_4K); + + if (!l2c310_base) + return -1; + + for (i = 0; i < L2C310_COUNTERS_NUM; i++) { + l2c310_counters[i].enabled = 0; + l2c310_counters[i].key = gator_events_get_key(); + } + + return gator_events_install(&gator_events_l2c310_interface); +} diff --git a/drivers/gator/gator_events_mali_4xx.c b/drivers/gator/gator_events_mali_4xx.c new file mode 100644 index 000000000000..45a42b212ae0 --- /dev/null +++ b/drivers/gator/gator_events_mali_4xx.c @@ -0,0 +1,624 @@ +/** + * Copyright (C) Arm Limited 2010-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include "gator.h" + +#include +#include +#include + +#include "linux/mali_linux_trace.h" + +#include "gator_events_mali_common.h" +#include "gator_events_mali_4xx.h" + +/* +* There have been four different variants of the comms between gator and Mali depending on driver version: +* # | DDK vsn range | Support | Notes +* +* 1 | (obsolete) | No software counter support | Obsolete patches +* 2 | (obsolete) | Tracepoint called for each separate s/w counter value as it appears | Obsolete patches +* 3 | r3p0-04rel0 - r3p2-01rel2 | Single tracepoint for all s/w counters in a bundle. | +* 4 | r3p2-01rel3 - date | As above but with extensions for MP devices (Mali-450) | At least r4p0-00rel1 +*/ + +#if !defined(GATOR_MALI_INTERFACE_STYLE) +#define GATOR_MALI_INTERFACE_STYLE (4) +#endif + +#if GATOR_MALI_INTERFACE_STYLE == 1 +#error GATOR_MALI_INTERFACE_STYLE 1 is obsolete +#elif GATOR_MALI_INTERFACE_STYLE == 2 +#error GATOR_MALI_INTERFACE_STYLE 2 is obsolete +#elif GATOR_MALI_INTERFACE_STYLE >= 3 +/* Valid GATOR_MALI_INTERFACE_STYLE */ +#else +#error Unknown GATOR_MALI_INTERFACE_STYLE option. +#endif + +#if GATOR_MALI_INTERFACE_STYLE < 4 +#include "mali/mali_mjollnir_profiling_gator_api.h" +#else +#include "mali/mali_utgard_profiling_gator_api.h" +#endif + +/* + * Check that the MALI_SUPPORT define is set to one of the allowable device codes. + */ +#if (MALI_SUPPORT != MALI_4xx) +#error MALI_SUPPORT set to an invalid device code: expecting MALI_4xx +#endif + +static const char mali_name[] = "4xx"; + +/* gatorfs variables for counter enable state, + * the event the counter should count and the + * 'key' (a unique id set by gatord and returned + * by gator.ko) + */ +static unsigned long counter_enabled[NUMBER_OF_EVENTS]; +static unsigned long counter_event[NUMBER_OF_EVENTS]; +static unsigned long counter_key[NUMBER_OF_EVENTS]; + +/* The data we have recorded */ +static atomic_t counter_data[NUMBER_OF_EVENTS]; + +/* An array used to return the data we recorded + * as key,value pairs hence the *2 + */ +static int counter_dump[NUMBER_OF_EVENTS * 2]; +static int counter_prev[NUMBER_OF_EVENTS]; +static bool prev_set[NUMBER_OF_EVENTS]; + +/* Note whether tracepoints have been registered */ +static int trace_registered; + +/* + * These numbers define the actual numbers of each block type that exist in the system. Initially + * these are set to the maxima defined above; if the driver is capable of being queried (newer + * drivers only) then the values may be revised. + */ +static unsigned int n_vp_cores = MAX_NUM_VP_CORES; +static unsigned int n_l2_cores = MAX_NUM_L2_CACHE_CORES; +static unsigned int n_fp_cores = MAX_NUM_FP_CORES; + +extern struct mali_counter mali_activity[2]; +static const char *const mali_activity_names[] = { + "fragment", + "vertex", +}; + +/** + * Returns non-zero if the given counter ID is an activity counter. + */ +static inline int is_activity_counter(unsigned int event_id) +{ + return (event_id >= FIRST_ACTIVITY_EVENT && + event_id <= LAST_ACTIVITY_EVENT); +} + +/** + * Returns non-zero if the given counter ID is a hardware counter. + */ +static inline int is_hw_counter(unsigned int event_id) +{ + return (event_id >= FIRST_HW_COUNTER && event_id <= LAST_HW_COUNTER); +} + +/* Probe for hardware counter events */ +GATOR_DEFINE_PROBE(mali_hw_counter, TP_PROTO(unsigned int event_id, unsigned int value)) +{ + if (is_hw_counter(event_id)) + atomic_add(value, &counter_data[event_id]); +} + +GATOR_DEFINE_PROBE(mali_sw_counters, TP_PROTO(pid_t pid, pid_t tid, void *surface_id, unsigned int *counters)) +{ + u32 i; + + /* Copy over the values for those counters which are enabled. */ + for (i = FIRST_SW_COUNTER; i <= LAST_SW_COUNTER; i++) { + if (counter_enabled[i]) + atomic_add(counters[i - FIRST_SW_COUNTER], &counter_data[i]); + } +} + +/** + * Create a single filesystem entry for a specified event. + * @param sb the superblock + * @param root Filesystem root + * @param name The name of the entry to create + * @param event The ID of the event + * @param create_event_item boolean indicating whether to create an 'event' filesystem entry. True to create. + * + * @return 0 if ok, non-zero if the create failed. + */ +static int create_fs_entry(struct super_block *sb, struct dentry *root, const char *name, int event, int create_event_item) +{ + struct dentry *dir; + + dir = gatorfs_mkdir(sb, root, name); + + if (!dir) + return -1; + + if (create_event_item) + gatorfs_create_ulong(sb, dir, "event", &counter_event[event]); + + gatorfs_create_ulong(sb, dir, "enabled", &counter_enabled[event]); + gatorfs_create_ro_ulong(sb, dir, "key", &counter_key[event]); + + return 0; +} + +#if GATOR_MALI_INTERFACE_STYLE > 3 +/* + * Read the version info structure if available + */ +static void initialise_version_info(void) +{ + void (*mali_profiling_get_mali_version_symbol)(struct _mali_profiling_mali_version *values); + + mali_profiling_get_mali_version_symbol = symbol_get(_mali_profiling_get_mali_version); + + if (mali_profiling_get_mali_version_symbol) { + struct _mali_profiling_mali_version version_info; + + pr_debug("gator: mali online _mali_profiling_get_mali_version symbol @ %p\n", + mali_profiling_get_mali_version_symbol); + + /* + * Revise the number of each different core type using information derived from the DDK. + */ + mali_profiling_get_mali_version_symbol(&version_info); + + n_fp_cores = version_info.num_of_fp_cores; + n_vp_cores = version_info.num_of_vp_cores; + n_l2_cores = version_info.num_of_l2_cores; + + /* Release the function - we're done with it. */ + symbol_put(_mali_profiling_get_mali_version); + } else { + pr_err("gator: mali online _mali_profiling_get_mali_version symbol not found\n"); + pr_err("gator: check your Mali DDK version versus the GATOR_MALI_INTERFACE_STYLE setting\n"); + } +} +#endif + +static int create_files(struct super_block *sb, struct dentry *root) +{ + int event; + + char buf[40]; + int core_id; + int counter_number; + + pr_debug("gator: Initialising counters with style = %d\n", GATOR_MALI_INTERFACE_STYLE); + +#if GATOR_MALI_INTERFACE_STYLE > 3 + /* + * Initialise first: this sets up the number of cores available (on compatible DDK versions). + * Ideally this would not need guarding but other parts of the code depend on the interface style being set + * correctly; if it is not then the system can enter an inconsistent state. + */ + initialise_version_info(); +#endif + + mali_activity[0].cores = n_fp_cores; + mali_activity[1].cores = n_vp_cores; + for (event = 0; event < ARRAY_SIZE(mali_activity); event++) { + if (gator_mali_create_file_system(mali_name, mali_activity_names[event], sb, root, &mali_activity[event], NULL) != 0) + return -1; + } + + /* Vertex processor counters */ + for (core_id = 0; core_id < n_vp_cores; core_id++) { + int activity_counter_id = ACTIVITY_VP_0; + + snprintf(buf, sizeof(buf), "ARM_Mali-%s_VP_%d_active", mali_name, core_id); + if (create_fs_entry(sb, root, buf, activity_counter_id, 0) != 0) + return -1; + + for (counter_number = 0; counter_number < 2; counter_number++) { + int counter_id = COUNTER_VP_0_C0 + (2 * core_id) + counter_number; + + snprintf(buf, sizeof(buf), "ARM_Mali-%s_VP_%d_cnt%d", mali_name, core_id, counter_number); + if (create_fs_entry(sb, root, buf, counter_id, 1) != 0) + return -1; + } + } + + /* Fragment processors' counters */ + for (core_id = 0; core_id < n_fp_cores; core_id++) { + int activity_counter_id = ACTIVITY_FP_0 + core_id; + + snprintf(buf, sizeof(buf), "ARM_Mali-%s_FP_%d_active", mali_name, core_id); + if (create_fs_entry(sb, root, buf, activity_counter_id, 0) != 0) + return -1; + + for (counter_number = 0; counter_number < 2; counter_number++) { + int counter_id = COUNTER_FP_0_C0 + (2 * core_id) + counter_number; + + snprintf(buf, sizeof(buf), "ARM_Mali-%s_FP_%d_cnt%d", mali_name, core_id, counter_number); + if (create_fs_entry(sb, root, buf, counter_id, 1) != 0) + return -1; + } + } + + /* L2 Cache counters */ + for (core_id = 0; core_id < n_l2_cores; core_id++) { + for (counter_number = 0; counter_number < 2; counter_number++) { + int counter_id = COUNTER_L2_0_C0 + (2 * core_id) + counter_number; + + snprintf(buf, sizeof(buf), "ARM_Mali-%s_L2_%d_cnt%d", mali_name, core_id, counter_number); + if (create_fs_entry(sb, root, buf, counter_id, 1) != 0) + return -1; + } + } + + /* Now set up the software counter entries */ + for (event = FIRST_SW_COUNTER; event <= LAST_SW_COUNTER; event++) { + snprintf(buf, sizeof(buf), "ARM_Mali-%s_SW_%d", mali_name, event - FIRST_SW_COUNTER); + + if (create_fs_entry(sb, root, buf, event, 0) != 0) + return -1; + } + + /* Now set up the special counter entries */ + snprintf(buf, sizeof(buf), "ARM_Mali-%s_Filmstrip_cnt0", mali_name); + if (create_fs_entry(sb, root, buf, COUNTER_FILMSTRIP, 1) != 0) + return -1; + +#ifdef DVFS_REPORTED_BY_DDK + snprintf(buf, sizeof(buf), "ARM_Mali-%s_Frequency", mali_name); + if (create_fs_entry(sb, root, buf, COUNTER_FREQUENCY, 1) != 0) + return -1; + + snprintf(buf, sizeof(buf), "ARM_Mali-%s_Voltage", mali_name); + if (create_fs_entry(sb, root, buf, COUNTER_VOLTAGE, 1) != 0) + return -1; +#endif + + return 0; +} + +/* + * Local store for the get_counters entry point into the DDK. + * This is stored here since it is used very regularly. + */ +static void (*mali_get_counters)(unsigned int *, unsigned int *, unsigned int *, unsigned int *); +static u32 (*mali_get_l2_counters)(struct _mali_profiling_l2_counter_values *values); + +/* + * Examine list of counters between two index limits and determine if any one is enabled. + * Returns 1 if any counter is enabled, 0 if none is. + */ +static int is_any_counter_enabled(unsigned int first_counter, unsigned int last_counter) +{ + unsigned int i; + + for (i = first_counter; i <= last_counter; i++) { + if (counter_enabled[i]) + return 1; /* At least one counter is enabled */ + } + + return 0; /* No s/w counters enabled */ +} + +static void init_counters(unsigned int from_counter, unsigned int to_counter) +{ + unsigned int counter_id; + + /* If a Mali driver is present and exporting the appropriate symbol + * then we can request the HW counters (of which there are only 2) + * be configured to count the desired events + */ + mali_profiling_set_event_type *mali_set_hw_event; + + mali_set_hw_event = symbol_get(_mali_profiling_set_event); + + if (mali_set_hw_event) { + pr_debug("gator: mali online _mali_profiling_set_event symbol @ %p\n", mali_set_hw_event); + + for (counter_id = from_counter; counter_id <= to_counter; counter_id++) { + if (counter_enabled[counter_id]) + mali_set_hw_event(counter_id, counter_event[counter_id]); + else + mali_set_hw_event(counter_id, 0xFFFFFFFF); + } + + symbol_put(_mali_profiling_set_event); + } else { + pr_err("gator: mali online _mali_profiling_set_event symbol not found\n"); + } +} + +static void mali_counter_initialize(void) +{ + int i; + + mali_profiling_control_type *mali_control; + + init_counters(COUNTER_L2_0_C0, COUNTER_L2_0_C0 + (2 * n_l2_cores) - 1); + init_counters(COUNTER_VP_0_C0, COUNTER_VP_0_C0 + (2 * n_vp_cores) - 1); + init_counters(COUNTER_FP_0_C0, COUNTER_FP_0_C0 + (2 * n_fp_cores) - 1); + + /* Generic control interface for Mali DDK. */ + mali_control = symbol_get(_mali_profiling_control); + if (mali_control) { + /* The event attribute in the XML file keeps the actual frame rate. */ + unsigned int rate = counter_event[COUNTER_FILMSTRIP] & 0xff; + unsigned int resize_factor = (counter_event[COUNTER_FILMSTRIP] >> 8) & 0xff; + + pr_debug("gator: mali online _mali_profiling_control symbol @ %p\n", mali_control); + + mali_control(SW_COUNTER_ENABLE, (is_any_counter_enabled(FIRST_SW_COUNTER, LAST_SW_COUNTER) ? 1 : 0)); + mali_control(FBDUMP_CONTROL_ENABLE, (counter_enabled[COUNTER_FILMSTRIP] ? 1 : 0)); + mali_control(FBDUMP_CONTROL_RATE, rate); + mali_control(FBDUMP_CONTROL_RESIZE_FACTOR, resize_factor); + + pr_debug("gator: sent mali_control enabled=%d, rate=%d\n", (counter_enabled[COUNTER_FILMSTRIP] ? 1 : 0), rate); + + symbol_put(_mali_profiling_control); + } else { + pr_err("gator: mali online _mali_profiling_control symbol not found\n"); + } + + mali_get_counters = symbol_get(_mali_profiling_get_counters); + if (mali_get_counters) + pr_debug("gator: mali online _mali_profiling_get_counters symbol @ %p\n", mali_get_counters); + else + pr_debug("gator: mali _mali_profiling_get_counters symbol not defined\n"); + + mali_get_l2_counters = symbol_get(_mali_profiling_get_l2_counters); + if (mali_get_l2_counters) + pr_debug("gator: mali online _mali_profiling_get_l2_counters symbol @ %p\n", mali_get_l2_counters); + else + pr_debug("gator: mali _mali_profiling_get_l2_counters symbol not defined\n"); + + if (!mali_get_counters && !mali_get_l2_counters) { + pr_err("gator: no L2 counters available\n"); + n_l2_cores = 0; + } + + /* Clear counters in the start */ + for (i = 0; i < NUMBER_OF_EVENTS; i++) { + atomic_set(&counter_data[i], 0); + prev_set[i] = false; + } +} + +static void mali_counter_deinitialize(void) +{ + mali_profiling_set_event_type *mali_set_hw_event; + mali_profiling_control_type *mali_control; + + mali_set_hw_event = symbol_get(_mali_profiling_set_event); + + if (mali_set_hw_event) { + int i; + + pr_debug("gator: mali offline _mali_profiling_set_event symbol @ %p\n", mali_set_hw_event); + for (i = FIRST_HW_COUNTER; i <= LAST_HW_COUNTER; i++) + mali_set_hw_event(i, 0xFFFFFFFF); + + symbol_put(_mali_profiling_set_event); + } else { + pr_err("gator: mali offline _mali_profiling_set_event symbol not found\n"); + } + + /* Generic control interface for Mali DDK. */ + mali_control = symbol_get(_mali_profiling_control); + + if (mali_control) { + pr_debug("gator: mali offline _mali_profiling_control symbol @ %p\n", mali_control); + + /* Reset the DDK state - disable counter collection */ + mali_control(SW_COUNTER_ENABLE, 0); + + mali_control(FBDUMP_CONTROL_ENABLE, 0); + + symbol_put(_mali_profiling_control); + } else { + pr_err("gator: mali offline _mali_profiling_control symbol not found\n"); + } + + if (mali_get_counters) + symbol_put(_mali_profiling_get_counters); + + if (mali_get_l2_counters) + symbol_put(_mali_profiling_get_l2_counters); +} + +static int start(void) +{ + /* register tracepoints */ + if (GATOR_REGISTER_TRACE(mali_hw_counter)) { + pr_err("gator: mali_hw_counter tracepoint failed to activate\n"); + return -1; + } + + /* For Mali drivers with built-in support. */ + if (GATOR_REGISTER_TRACE(mali_sw_counters)) { + pr_err("gator: mali_sw_counters tracepoint failed to activate\n"); + return -1; + } + + trace_registered = 1; + + mali_counter_initialize(); + return 0; +} + +static void stop(void) +{ + unsigned int cnt; + + pr_debug("gator: mali stop\n"); + + if (trace_registered) { + GATOR_UNREGISTER_TRACE(mali_hw_counter); + + /* For Mali drivers with built-in support. */ + GATOR_UNREGISTER_TRACE(mali_sw_counters); + + pr_debug("gator: mali timeline tracepoint deactivated\n"); + + trace_registered = 0; + } + + for (cnt = 0; cnt < NUMBER_OF_EVENTS; cnt++) { + counter_enabled[cnt] = 0; + counter_event[cnt] = 0; + } + + mali_counter_deinitialize(); +} + +static void dump_counters(unsigned int from_counter, unsigned int to_counter, unsigned int *len) +{ + unsigned int counter_id; + + for (counter_id = from_counter; counter_id <= to_counter; counter_id++) { + if (counter_enabled[counter_id]) { + unsigned int value; + + counter_dump[(*len)++] = counter_key[counter_id]; + value = atomic_read(&counter_data[counter_id]); + atomic_sub(value, &counter_data[counter_id]); + counter_dump[(*len)++] = value; + } + } +} + +static int read(int **buffer, bool sched_switch) +{ + int len = 0; + + if (!on_primary_core()) + return 0; + + /* Read the L2 C0 and C1 here. */ + if (n_l2_cores > 0 && is_any_counter_enabled(COUNTER_L2_0_C0, COUNTER_L2_0_C0 + (2 * n_l2_cores))) { + unsigned int unavailable_l2_caches = 0; + struct _mali_profiling_l2_counter_values cache_values; + unsigned int cache_id; + struct _mali_profiling_core_counters *per_core; + + /* Poke the driver to get the counter values - older style; only one L2 cache */ + if (mali_get_l2_counters) { + unavailable_l2_caches = mali_get_l2_counters(&cache_values); + } else if (mali_get_counters) { + per_core = &cache_values.cores[0]; + mali_get_counters(&per_core->source0, &per_core->value0, &per_core->source1, &per_core->value1); + } else { + /* This should never happen, as n_l2_caches is only set > 0 if one of the above functions is found. */ + } + + /* Fill in the two cache counter values for each cache block. */ + for (cache_id = 0; cache_id < n_l2_cores; cache_id++) { + unsigned int counter_id_0 = COUNTER_L2_0_C0 + (2 * cache_id); + unsigned int counter_id_1 = counter_id_0 + 1; + + if ((1 << cache_id) & unavailable_l2_caches) + continue; /* This cache is unavailable (powered-off, possibly). */ + + per_core = &cache_values.cores[cache_id]; + + if (counter_enabled[counter_id_0] && prev_set[counter_id_0]) { + /* Calculate and save src0's counter val0 */ + counter_dump[len++] = counter_key[counter_id_0]; + counter_dump[len++] = per_core->value0 - counter_prev[counter_id_0]; + } + + if (counter_enabled[counter_id_1] && prev_set[counter_id_1]) { + /* Calculate and save src1's counter val1 */ + counter_dump[len++] = counter_key[counter_id_1]; + counter_dump[len++] = per_core->value1 - counter_prev[counter_id_1]; + } + + /* Save the previous values for the counters. */ + counter_prev[counter_id_0] = per_core->value0; + prev_set[counter_id_0] = true; + counter_prev[counter_id_1] = per_core->value1; + prev_set[counter_id_1] = true; + } + } + + /* Process other (non-timeline) counters. */ + dump_counters(COUNTER_VP_0_C0, COUNTER_VP_0_C0 + (2 * n_vp_cores) - 1, &len); + dump_counters(COUNTER_FP_0_C0, COUNTER_FP_0_C0 + (2 * n_fp_cores) - 1, &len); + + dump_counters(FIRST_SW_COUNTER, LAST_SW_COUNTER, &len); + +#ifdef DVFS_REPORTED_BY_DDK + { + int cnt; + /* + * Add in the voltage and frequency counters if enabled. Note + * that, since these are actually passed as events, the counter + * value should not be cleared. + * + * Intentionally use atomic_read as these are absolute counters + */ + cnt = COUNTER_FREQUENCY; + if (counter_enabled[cnt]) { + counter_dump[len++] = counter_key[cnt]; + counter_dump[len++] = atomic_read(&counter_data[cnt]); + } + + cnt = COUNTER_VOLTAGE; + if (counter_enabled[cnt]) { + counter_dump[len++] = counter_key[cnt]; + counter_dump[len++] = atomic_read(&counter_data[cnt]); + } + } +#endif + + if (buffer) + *buffer = counter_dump; + + return len; +} + +static struct gator_interface gator_events_mali_interface = { + .name = "mali_4xx", + .create_files = create_files, + .start = start, + .stop = stop, + .read = read, +}; + +extern void gator_events_mali_log_dvfs_event(unsigned int frequency_mhz, unsigned int voltage_mv) +{ +#ifdef DVFS_REPORTED_BY_DDK + /* Intentionally use atomic_set as these are absolute counters */ + atomic_set(&counter_data[COUNTER_FREQUENCY], frequency_mhz); + atomic_set(&counter_data[COUNTER_VOLTAGE], voltage_mv); +#endif +} + +int gator_events_mali_init(void) +{ + unsigned int cnt; + + pr_debug("gator: mali init\n"); + + gator_mali_initialise_counters(mali_activity, ARRAY_SIZE(mali_activity)); + + for (cnt = 0; cnt < NUMBER_OF_EVENTS; cnt++) { + counter_enabled[cnt] = 0; + counter_event[cnt] = 0; + counter_key[cnt] = gator_events_get_key(); + atomic_set(&counter_data[cnt], 0); + } + + trace_registered = 0; + + return gator_events_install(&gator_events_mali_interface); +} diff --git a/drivers/gator/gator_events_mali_4xx.h b/drivers/gator/gator_events_mali_4xx.h new file mode 100644 index 000000000000..ee20c22234fa --- /dev/null +++ b/drivers/gator/gator_events_mali_4xx.h @@ -0,0 +1,18 @@ +/** + * Copyright (C) Arm Limited 2011-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +/* + * Header contains common definitions for the Mali-4xx processors. + */ +#if !defined(GATOR_EVENTS_MALI_4xx_H) +#define GATOR_EVENTS_MALI_4xx_H + +extern void gator_events_mali_log_dvfs_event(unsigned int d0, unsigned int d1); + +#endif /* GATOR_EVENTS_MALI_4xx_H */ diff --git a/drivers/gator/gator_events_mali_common.c b/drivers/gator/gator_events_mali_common.c new file mode 100644 index 000000000000..925ce98bafbf --- /dev/null +++ b/drivers/gator/gator_events_mali_common.c @@ -0,0 +1,72 @@ +/** + * Copyright (C) Arm Limited 2012-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ +#include "gator_events_mali_common.h" + +extern int gator_mali_create_file_system(const char *mali_name, const char *event_name, struct super_block *sb, struct dentry *root, struct mali_counter *counter, unsigned long *event) +{ + int err; + char buf[50]; + struct dentry *dir; + + /* If the counter name is empty ignore it */ + if (strlen(event_name) != 0) { + /* Set up the filesystem entry for this event. */ + if (mali_name == NULL) + snprintf(buf, sizeof(buf), "ARM_Mali-%s", event_name); + else + snprintf(buf, sizeof(buf), "ARM_Mali-%s_%s", mali_name, event_name); + + dir = gatorfs_mkdir(sb, root, buf); + + if (dir == NULL) { + pr_err("gator: %s: error creating file system for: %s (%s)\n", mali_name, event_name, buf); + return -1; + } + + err = gatorfs_create_ulong(sb, dir, "enabled", &counter->enabled); + if (err != 0) { + pr_err("gator: %s: error calling gatorfs_create_ulong for: %s (%s)\n", mali_name, event_name, buf); + return -1; + } + err = gatorfs_create_ro_ulong(sb, dir, "key", &counter->key); + if (err != 0) { + pr_err("gator: %s: error calling gatorfs_create_ro_ulong for: %s (%s)\n", mali_name, event_name, buf); + return -1; + } + if (counter->cores != -1) { + err = gatorfs_create_ro_ulong(sb, dir, "cores", &counter->cores); + if (err != 0) { + pr_err("gator: %s: error calling gatorfs_create_ro_ulong for: %s (%s)\n", mali_name, event_name, buf); + return -1; + } + } + if (event != NULL) { + err = gatorfs_create_ulong(sb, dir, "event", event); + if (err != 0) { + pr_err("gator: %s: error calling gatorfs_create_ro_ulong for: %s (%s)\n", mali_name, event_name, buf); + return -1; + } + } + } + + return 0; +} + +extern void gator_mali_initialise_counters(struct mali_counter counters[], unsigned int n_counters) +{ + unsigned int cnt; + + for (cnt = 0; cnt < n_counters; cnt++) { + struct mali_counter *counter = &counters[cnt]; + + counter->key = gator_events_get_key(); + counter->enabled = 0; + counter->cores = -1; + } +} diff --git a/drivers/gator/gator_events_mali_common.h b/drivers/gator/gator_events_mali_common.h new file mode 100644 index 000000000000..75d22a2351c4 --- /dev/null +++ b/drivers/gator/gator_events_mali_common.h @@ -0,0 +1,77 @@ +/** + * Copyright (C) Arm Limited 2012-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#if !defined(GATOR_EVENTS_MALI_COMMON_H) +#define GATOR_EVENTS_MALI_COMMON_H + +#include "gator.h" + +#include +#include +#include +#include +#include + +/* Ensure that MALI_SUPPORT has been defined to something. */ +#ifndef MALI_SUPPORT +#error MALI_SUPPORT not defined! +#endif + +/* Values for the supported activity event types */ +#define ACTIVITY_START (1) +#define ACTIVITY_STOP (2) + +/* + * Runtime state information for a counter. + */ +struct mali_counter { + /* 'key' (a unique id set by gatord and returned by gator.ko) */ + unsigned long key; + /* counter enable state */ + unsigned long enabled; + /* for activity counters, the number of cores, otherwise -1 */ + unsigned long cores; +}; + +/* + * Mali-4xx + */ +typedef int mali_profiling_set_event_type(unsigned int, int); +typedef void mali_profiling_control_type(unsigned int, unsigned int); + +/* + * Driver entry points for functions called directly by gator. + */ +extern int _mali_profiling_set_event(unsigned int, int); +extern void _mali_profiling_control(unsigned int, unsigned int); +extern void _mali_profiling_get_counters(unsigned int *, unsigned int *, unsigned int *, unsigned int *); + +/** + * Creates a filesystem entry under /dev/gator relating to the specified event name and key, and + * associate the key/enable values with this entry point. + * + * @param event_name The name of the event. + * @param sb Linux super block + * @param root Directory under which the entry will be created. + * @param counter_key Ptr to location which will be associated with the counter key. + * @param counter_enabled Ptr to location which will be associated with the counter enable state. + * + * @return 0 if entry point was created, non-zero if not. + */ +extern int gator_mali_create_file_system(const char *mali_name, const char *event_name, struct super_block *sb, struct dentry *root, struct mali_counter *counter, unsigned long *event); + +/** + * Initializes the counter array. + * + * @param keys The array of counters + * @param n_counters The number of entries in each of the arrays. + */ +extern void gator_mali_initialise_counters(struct mali_counter counters[], unsigned int n_counters); + +#endif /* GATOR_EVENTS_MALI_COMMON_H */ diff --git a/drivers/gator/gator_events_mali_midgard.c b/drivers/gator/gator_events_mali_midgard.c new file mode 100644 index 000000000000..1c5cef4b85f7 --- /dev/null +++ b/drivers/gator/gator_events_mali_midgard.c @@ -0,0 +1,570 @@ +/** + * Copyright (C) Arm Limited 2011-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#include "gator.h" + +#include +#include +#include +#include +#include + +#include "gator_events_mali_midgard.h" + +#ifdef MALI_DIR_MIDGARD +/* New DDK Directory structure with kernel/drivers/gpu/arm/midgard*/ +#include "mali_linux_trace.h" +#else +/* Old DDK Directory structure with kernel/drivers/gpu/arm/t6xx*/ +#include "linux/mali_linux_trace.h" +#endif + +#include "gator_events_mali_common.h" + +/* + * Check that the MALI_SUPPORT define is set to one of the allowable device codes. + */ +#if (MALI_SUPPORT != MALI_MIDGARD_OR_BIFROST) +#error MALI_SUPPORT set to an invalid device code: expecting MALI_MIDGARD_OR_BIFROST +#endif + +// This name has to be set in the runtime, since it might be Midgard or Bifrost, depending on circumstances. +const char* mali_name = NULL; + +/* Counters for Mali-Midgard/Bifrost: + * + * - Timeline events + * They are tracepoints, but instead of reporting a number they report a START/STOP event. + * They are reported in Streamline as number of microseconds while that particular counter was active. + * + * - SW counters + * They are tracepoints reporting a particular number. + * They are accumulated in sw_counter_data array until they are passed to Streamline, then they are zeroed. + * + * - Accumulators + * They are the same as software counters but their value is not zeroed. + */ + +/* Timeline (start/stop) activity */ +static const char *const timeline_event_names[] = { + "PM_SHADER_0", + "PM_SHADER_1", + "PM_SHADER_2", + "PM_SHADER_3", + "PM_SHADER_4", + "PM_SHADER_5", + "PM_SHADER_6", + "PM_SHADER_7", + "PM_TILER_0", + "PM_L2_0", + "PM_L2_1", + "MMU_AS_0", + "MMU_AS_1", + "MMU_AS_2", + "MMU_AS_3" +}; + +enum { + PM_SHADER_0 = 0, + PM_SHADER_1, + PM_SHADER_2, + PM_SHADER_3, + PM_SHADER_4, + PM_SHADER_5, + PM_SHADER_6, + PM_SHADER_7, + PM_TILER_0, + PM_L2_0, + PM_L2_1, + MMU_AS_0, + MMU_AS_1, + MMU_AS_2, + MMU_AS_3 +}; +/* The number of shader blocks in the enum above */ +#define NUM_PM_SHADER (8) + +/* Software Counters */ +static const char *const software_counter_names[] = { + "MMU_PAGE_FAULT_0", + "MMU_PAGE_FAULT_1", + "MMU_PAGE_FAULT_2", + "MMU_PAGE_FAULT_3" +}; + +enum { + MMU_PAGE_FAULT_0 = 0, + MMU_PAGE_FAULT_1, + MMU_PAGE_FAULT_2, + MMU_PAGE_FAULT_3 +}; + +/* Software Counters */ +static const char *const accumulators_names[] = { + "TOTAL_ALLOC_PAGES" +}; + +enum { + TOTAL_ALLOC_PAGES = 0 +}; + +#define FIRST_TIMELINE_EVENT (0) +#define NUMBER_OF_TIMELINE_EVENTS (sizeof(timeline_event_names) / sizeof(timeline_event_names[0])) +#define FIRST_SOFTWARE_COUNTER (FIRST_TIMELINE_EVENT + NUMBER_OF_TIMELINE_EVENTS) +#define NUMBER_OF_SOFTWARE_COUNTERS (sizeof(software_counter_names) / sizeof(software_counter_names[0])) +#define FIRST_ACCUMULATOR (FIRST_SOFTWARE_COUNTER + NUMBER_OF_SOFTWARE_COUNTERS) +#define NUMBER_OF_ACCUMULATORS (sizeof(accumulators_names) / sizeof(accumulators_names[0])) +#define FILMSTRIP (NUMBER_OF_TIMELINE_EVENTS + NUMBER_OF_SOFTWARE_COUNTERS + NUMBER_OF_ACCUMULATORS) +#define NUMBER_OF_EVENTS (NUMBER_OF_TIMELINE_EVENTS + NUMBER_OF_SOFTWARE_COUNTERS + NUMBER_OF_ACCUMULATORS + 1) + +/* + * gatorfs variables for counter enable state + */ +static struct mali_counter counters[NUMBER_OF_EVENTS]; +static unsigned long filmstrip_event; + +/* An array used to return the data we recorded + * as key,value pairs hence the *2 + */ +static int counter_dump[NUMBER_OF_EVENTS * 2]; + +/* + * Array holding counter start times (in ns) for each counter. A zero + * here indicates that the activity monitored by this counter is not + * running. + */ +static struct timespec timeline_event_starttime[NUMBER_OF_TIMELINE_EVENTS]; + +/* The data we have recorded */ +static unsigned int timeline_data[NUMBER_OF_TIMELINE_EVENTS]; +static unsigned int sw_counter_data[NUMBER_OF_SOFTWARE_COUNTERS]; +static unsigned int accumulators_data[NUMBER_OF_ACCUMULATORS]; + +/* Hold the previous timestamp, used to calculate the sample interval. */ +static struct timespec prev_timestamp; + +static unsigned long long previous_shader_bitmask; +static unsigned long long previous_tiler_bitmask; +static unsigned long long previous_l2_bitmask; + +/** + * Returns the timespan (in microseconds) between the two specified timestamps. + * + * @param start Ptr to the start timestamp + * @param end Ptr to the end timestamp + * + * @return Number of microseconds between the two timestamps (can be negative if start follows end). + */ +static inline long get_duration_us(const struct timespec *start, const struct timespec *end) +{ + long event_duration_us = (end->tv_nsec - start->tv_nsec) / 1000; + + event_duration_us += (end->tv_sec - start->tv_sec) * 1000000; + + return event_duration_us; +} + +static void record_timeline_event(unsigned int timeline_index, unsigned int type) +{ + struct timespec event_timestamp; + struct timespec *event_start = &timeline_event_starttime[timeline_index]; + + switch (type) { + case ACTIVITY_START: + /* Get the event time... */ + getnstimeofday(&event_timestamp); + + /* Remember the start time if the activity is not already started */ + if (event_start->tv_sec == 0) + *event_start = event_timestamp; /* Structure copy */ + break; + + case ACTIVITY_STOP: + /* if the counter was started... */ + if (event_start->tv_sec != 0) { + /* Get the event time... */ + getnstimeofday(&event_timestamp); + + /* Accumulate the duration in us */ + timeline_data[timeline_index] += get_duration_us(event_start, &event_timestamp); + + /* Reset the start time to indicate the activity is stopped. */ + event_start->tv_sec = 0; + } + break; + + default: + /* Other activity events are ignored. */ + break; + } +} + +/* + * Documentation about the following tracepoints is in mali_linux_trace.h + */ + +GATOR_DEFINE_PROBE(mali_pm_status, TP_PROTO(unsigned int event_id, unsigned long long value)) +{ +#define SHADER_PRESENT_LO 0x100 /* (RO) Shader core present bitmap, low word */ +#define TILER_PRESENT_LO 0x110 /* (RO) Tiler core present bitmap, low word */ +#define L2_PRESENT_LO 0x120 /* (RO) Level 2 cache present bitmap, low word */ +#define BIT_AT(value, pos) ((value >> pos) & 1) + + switch (event_id) { + case SHADER_PRESENT_LO: + { + unsigned long long changed_bitmask = previous_shader_bitmask ^ value; + int pos; + + for (pos = 0; pos < NUM_PM_SHADER; ++pos) { + if (BIT_AT(changed_bitmask, pos)) + record_timeline_event(PM_SHADER_0 + pos, BIT_AT(value, pos) ? ACTIVITY_START : ACTIVITY_STOP); + } + + previous_shader_bitmask = value; + break; + } + + case TILER_PRESENT_LO: + { + unsigned long long changed = previous_tiler_bitmask ^ value; + + if (BIT_AT(changed, 0)) + record_timeline_event(PM_TILER_0, BIT_AT(value, 0) ? ACTIVITY_START : ACTIVITY_STOP); + + previous_tiler_bitmask = value; + break; + } + + case L2_PRESENT_LO: + { + unsigned long long changed = previous_l2_bitmask ^ value; + + if (BIT_AT(changed, 0)) + record_timeline_event(PM_L2_0, BIT_AT(value, 0) ? ACTIVITY_START : ACTIVITY_STOP); + if (BIT_AT(changed, 4)) + record_timeline_event(PM_L2_1, BIT_AT(value, 4) ? ACTIVITY_START : ACTIVITY_STOP); + + previous_l2_bitmask = value; + break; + } + + default: + /* No other blocks are supported at present */ + break; + } + +#undef SHADER_PRESENT_LO +#undef TILER_PRESENT_LO +#undef L2_PRESENT_LO +#undef BIT_AT +} + +GATOR_DEFINE_PROBE(mali_page_fault_insert_pages, TP_PROTO(int event_id, unsigned long value)) +{ + /* We add to the previous since we may receive many tracepoints in one sample period */ + sw_counter_data[MMU_PAGE_FAULT_0 + event_id] += value; +} + +GATOR_DEFINE_PROBE(mali_mmu_as_in_use, TP_PROTO(int event_id)) +{ + record_timeline_event(MMU_AS_0 + event_id, ACTIVITY_START); +} + +GATOR_DEFINE_PROBE(mali_mmu_as_released, TP_PROTO(int event_id)) +{ + record_timeline_event(MMU_AS_0 + event_id, ACTIVITY_STOP); +} + +GATOR_DEFINE_PROBE(mali_total_alloc_pages_change, TP_PROTO(long long int event_id)) +{ + accumulators_data[TOTAL_ALLOC_PAGES] = event_id; +} + +static int create_files(struct super_block *sb, struct dentry *root) +{ + int event; + /* + * Create the filesystem for all events + */ + int counter_index = 0; + mali_profiling_control_type *mali_control; + + for (event = FIRST_TIMELINE_EVENT; event < FIRST_TIMELINE_EVENT + NUMBER_OF_TIMELINE_EVENTS; event++) { + if (gator_mali_create_file_system(mali_name, timeline_event_names[counter_index], sb, root, &counters[event], NULL) != 0) + return -1; + counter_index++; + } + counter_index = 0; + for (event = FIRST_SOFTWARE_COUNTER; event < FIRST_SOFTWARE_COUNTER + NUMBER_OF_SOFTWARE_COUNTERS; event++) { + if (gator_mali_create_file_system(mali_name, software_counter_names[counter_index], sb, root, &counters[event], NULL) != 0) + return -1; + counter_index++; + } + counter_index = 0; + for (event = FIRST_ACCUMULATOR; event < FIRST_ACCUMULATOR + NUMBER_OF_ACCUMULATORS; event++) { + if (gator_mali_create_file_system(mali_name, accumulators_names[counter_index], sb, root, &counters[event], NULL) != 0) + return -1; + counter_index++; + } + + mali_control = symbol_get(_mali_profiling_control); + if (mali_control) { + if (gator_mali_create_file_system(mali_name, "Filmstrip_cnt0", sb, root, &counters[FILMSTRIP], &filmstrip_event) != 0) + return -1; + symbol_put(_mali_profiling_control); + } + + return 0; +} + +static int register_tracepoints(void) +{ + if (GATOR_REGISTER_TRACE(mali_pm_status)) { + pr_err("gator: %s: mali_pm_status tracepoint failed to activate\n", mali_name); + return 0; + } + + if (GATOR_REGISTER_TRACE(mali_page_fault_insert_pages)) { + pr_err("gator: %s: mali_page_fault_insert_pages tracepoint failed to activate\n", mali_name); + return 0; + } + + if (GATOR_REGISTER_TRACE(mali_mmu_as_in_use)) { + pr_err("gator: %s: mali_mmu_as_in_use tracepoint failed to activate\n", mali_name); + return 0; + } + + if (GATOR_REGISTER_TRACE(mali_mmu_as_released)) { + pr_err("gator: %s: mali_mmu_as_released tracepoint failed to activate\n", mali_name); + return 0; + } + + if (GATOR_REGISTER_TRACE(mali_total_alloc_pages_change)) { + pr_err("gator: %s: mali_total_alloc_pages_change tracepoint failed to activate\n", mali_name); + return 0; + } + + pr_debug("gator: %s: start\n", mali_name); + pr_debug("gator: %s: mali_pm_status probe is at %p\n", mali_name, &probe_mali_pm_status); + pr_debug("gator: %s: mali_page_fault_insert_pages probe is at %p\n", mali_name, &probe_mali_page_fault_insert_pages); + pr_debug("gator: %s: mali_mmu_as_in_use probe is at %p\n", mali_name, &probe_mali_mmu_as_in_use); + pr_debug("gator: %s: mali_mmu_as_released probe is at %p\n", mali_name, &probe_mali_mmu_as_released); + pr_debug("gator: %s: mali_total_alloc_pages_change probe is at %p\n", mali_name, &probe_mali_total_alloc_pages_change); + + return 1; +} + +static int start(void) +{ + unsigned int cnt; + mali_profiling_control_type *mali_control; + + previous_shader_bitmask = 0; + previous_tiler_bitmask = 0; + previous_l2_bitmask = 0; + + /* Clean all data for the next capture */ + for (cnt = 0; cnt < NUMBER_OF_TIMELINE_EVENTS; cnt++) { + timeline_event_starttime[cnt].tv_sec = timeline_event_starttime[cnt].tv_nsec = 0; + timeline_data[cnt] = 0; + } + + for (cnt = 0; cnt < NUMBER_OF_SOFTWARE_COUNTERS; cnt++) + sw_counter_data[cnt] = 0; + + for (cnt = 0; cnt < NUMBER_OF_ACCUMULATORS; cnt++) + accumulators_data[cnt] = 0; + + /* Register tracepoints */ + if (register_tracepoints() == 0) + return -1; + + /* Generic control interface for Mali DDK. */ + mali_control = symbol_get(_mali_profiling_control); + if (mali_control) { + /* The event attribute in the XML file keeps the actual frame rate. */ + unsigned int enabled = counters[FILMSTRIP].enabled ? 1 : 0; + unsigned int rate = filmstrip_event & 0xff; + unsigned int resize_factor = (filmstrip_event >> 8) & 0xff; + + pr_debug("gator: mali online _mali_profiling_control symbol @ %p\n", mali_control); + +#define FBDUMP_CONTROL_ENABLE (1) +#define FBDUMP_CONTROL_RATE (2) +#define FBDUMP_CONTROL_RESIZE_FACTOR (4) + mali_control(FBDUMP_CONTROL_ENABLE, enabled); + mali_control(FBDUMP_CONTROL_RATE, rate); + mali_control(FBDUMP_CONTROL_RESIZE_FACTOR, resize_factor); + + pr_debug("gator: sent mali_control enabled=%d, rate=%d, resize_factor=%d\n", enabled, rate, resize_factor); + + symbol_put(_mali_profiling_control); + } else { + pr_err("gator: mali online _mali_profiling_control symbol not found\n"); + } + + /* + * Set the first timestamp for calculating the sample interval. The first interval could be quite long, + * since it will be the time between 'start' and the first 'read'. + * This means that timeline values will be divided by a big number for the first sample. + */ + getnstimeofday(&prev_timestamp); + + return 0; +} + +static void stop(void) +{ + mali_profiling_control_type *mali_control; + + pr_debug("gator: %s: stop\n", mali_name); + + /* + * It is safe to unregister traces even if they were not successfully + * registered, so no need to check. + */ + GATOR_UNREGISTER_TRACE(mali_pm_status); + pr_debug("gator: %s: mali_pm_status tracepoint deactivated\n", mali_name); + + GATOR_UNREGISTER_TRACE(mali_page_fault_insert_pages); + pr_debug("gator: %s: mali_page_fault_insert_pages tracepoint deactivated\n", mali_name); + + GATOR_UNREGISTER_TRACE(mali_mmu_as_in_use); + pr_debug("gator: %s: mali_mmu_as_in_use tracepoint deactivated\n", mali_name); + + GATOR_UNREGISTER_TRACE(mali_mmu_as_released); + pr_debug("gator: %s: mali_mmu_as_released tracepoint deactivated\n", mali_name); + + GATOR_UNREGISTER_TRACE(mali_total_alloc_pages_change); + pr_debug("gator: %s: mali_total_alloc_pages_change tracepoint deactivated\n", mali_name); + + /* Generic control interface for Mali DDK. */ + mali_control = symbol_get(_mali_profiling_control); + if (mali_control) { + pr_debug("gator: mali offline _mali_profiling_control symbol @ %p\n", mali_control); + + mali_control(FBDUMP_CONTROL_ENABLE, 0); + + symbol_put(_mali_profiling_control); + } else { + pr_err("gator: mali offline _mali_profiling_control symbol not found\n"); + } +} + +static int read(int **buffer, bool sched_switch) +{ + int cnt; + int len = 0; + long sample_interval_us = 0; + struct timespec read_timestamp; + + if (!on_primary_core()) + return 0; + + /* Get the start of this sample period. */ + getnstimeofday(&read_timestamp); + + /* + * Calculate the sample interval if the previous sample time is valid. + * We use tv_sec since it will not be 0. + */ + if (prev_timestamp.tv_sec != 0) + sample_interval_us = get_duration_us(&prev_timestamp, &read_timestamp); + + /* Structure copy. Update the previous timestamp. */ + prev_timestamp = read_timestamp; + + /* + * Report the timeline counters (ACTIVITY_START/STOP) + */ + for (cnt = FIRST_TIMELINE_EVENT; cnt < (FIRST_TIMELINE_EVENT + NUMBER_OF_TIMELINE_EVENTS); cnt++) { + struct mali_counter *counter = &counters[cnt]; + + if (counter->enabled) { + const int index = cnt - FIRST_TIMELINE_EVENT; + unsigned int value; + + /* If the activity is still running, reset its start time to the + * start of this sample period to correct the count. Add the + * time up to the end of the sample onto the count. + */ + if (timeline_event_starttime[index].tv_sec != 0) { + const long event_duration = get_duration_us(&timeline_event_starttime[index], &read_timestamp); + + timeline_data[index] += event_duration; + timeline_event_starttime[index] = read_timestamp; /* Activity is still running. */ + } + + if (sample_interval_us != 0) { + /* Convert the counter to a percent-of-sample value */ + value = (timeline_data[index] * 100) / sample_interval_us; + } else { + pr_debug("gator: %s: setting value to zero\n", mali_name); + value = 0; + } + + /* Clear the counter value ready for the next sample. */ + timeline_data[index] = 0; + + counter_dump[len++] = counter->key; + counter_dump[len++] = value; + } + } + + /* Report the software counters */ + for (cnt = FIRST_SOFTWARE_COUNTER; cnt < (FIRST_SOFTWARE_COUNTER + NUMBER_OF_SOFTWARE_COUNTERS); cnt++) { + const struct mali_counter *counter = &counters[cnt]; + + if (counter->enabled) { + const int index = cnt - FIRST_SOFTWARE_COUNTER; + + counter_dump[len++] = counter->key; + counter_dump[len++] = sw_counter_data[index]; + /* Set the value to zero for the next time */ + sw_counter_data[index] = 0; + } + } + + /* Report the accumulators */ + for (cnt = FIRST_ACCUMULATOR; cnt < (FIRST_ACCUMULATOR + NUMBER_OF_ACCUMULATORS); cnt++) { + const struct mali_counter *counter = &counters[cnt]; + + if (counter->enabled) { + const int index = cnt - FIRST_ACCUMULATOR; + + counter_dump[len++] = counter->key; + counter_dump[len++] = accumulators_data[index]; + /* Do not zero the accumulator */ + } + } + + /* Update the buffer */ + if (buffer) + *buffer = counter_dump; + + return len; +} + +static struct gator_interface gator_events_mali_midgard_interface = { + .name = "mali_sw_counters", + .create_files = create_files, + .start = start, + .stop = stop, + .read = read +}; + +extern int gator_events_mali_midgard_init(void) +{ + pr_debug("gator: Mali-%s: sw_counters init\n", mali_name); + + gator_mali_initialise_counters(counters, NUMBER_OF_EVENTS); + + return gator_events_install(&gator_events_mali_midgard_interface); +} diff --git a/drivers/gator/gator_events_mali_midgard.h b/drivers/gator/gator_events_mali_midgard.h new file mode 100644 index 000000000000..1d68a681323c --- /dev/null +++ b/drivers/gator/gator_events_mali_midgard.h @@ -0,0 +1,38 @@ +/** + * Copyright (C) Arm Limited 2013-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef GATOR_EVENTS_MALI_MIDGARD_H_ +#define GATOR_EVENTS_MALI_MIDGARD_H_ + +#define SYMBOL_GET(FUNCTION, ERROR_COUNT) \ + do { \ + if (FUNCTION ## _symbol) { \ + pr_err("gator: mali " #FUNCTION " symbol was already registered\n"); \ + (ERROR_COUNT)++; \ + } else { \ + FUNCTION ## _symbol = symbol_get(FUNCTION); \ + if (!FUNCTION ## _symbol) { \ + pr_err("gator: mali online " #FUNCTION " symbol not found\n"); \ + (ERROR_COUNT)++; \ + } \ + } \ + } while (0) + +#define SYMBOL_CLEANUP(FUNCTION) \ + do { \ + if (FUNCTION ## _symbol) { \ + symbol_put(FUNCTION); \ + FUNCTION ## _symbol = NULL; \ + } \ + } while (0) + + +// We're using a single place where name for mali GPU architecture is defined. +extern const char* mali_name; + +#endif /* GATOR_EVENTS_MALI_MIDGARD_H_ */ diff --git a/drivers/gator/gator_events_mali_midgard_hw.c b/drivers/gator/gator_events_mali_midgard_hw.c new file mode 100644 index 000000000000..67351ad7254b --- /dev/null +++ b/drivers/gator/gator_events_mali_midgard_hw.c @@ -0,0 +1,1026 @@ +/** + * Copyright (C) Arm Limited 2012-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#include "gator.h" + +#include +#include +#include +#include +#include + +/* Mali Midgard/Bifrost DDK includes */ +#if defined(MALI_SIMPLE_API) +/* Header with wrapper functions to kbase structures and functions */ +#include "mali_kbase_gator_api.h" +#elif defined(MALI_DIR_MIDGARD) +/* New DDK Directory structure with kernel/drivers/gpu/arm/midgard */ +#include "mali_linux_trace.h" +#include "mali_kbase.h" +#include "mali_kbase_mem_linux.h" +#else +/* Old DDK Directory structure with kernel/drivers/gpu/arm/t6xx */ +#include "linux/mali_linux_trace.h" +#include "kbase/src/common/mali_kbase.h" +#include "kbase/src/linux/mali_kbase_mem_linux.h" +#endif + +/* If API version is not specified then assume API version 1. */ +#ifndef MALI_DDK_GATOR_API_VERSION +#define MALI_DDK_GATOR_API_VERSION 1 +#endif + +#if (MALI_DDK_GATOR_API_VERSION != 1) && (MALI_DDK_GATOR_API_VERSION != 2) && (MALI_DDK_GATOR_API_VERSION != 3) +#error MALI_DDK_GATOR_API_VERSION is invalid (must be 1 for r1/r2 DDK, or 2 for r3/r4 DDK, or 3 for r5 and later DDK). +#endif + +#include "gator_events_mali_common.h" + +/* + * Mali Midgard or Bifrost + */ +#if MALI_DDK_GATOR_API_VERSION == 3 +static uint32_t (*kbase_gator_instr_hwcnt_dump_irq_symbol)(struct kbase_gator_hwcnt_handles *); +static uint32_t (*kbase_gator_instr_hwcnt_dump_complete_symbol)(struct kbase_gator_hwcnt_handles *, uint32_t *const); +static struct kbase_gator_hwcnt_handles *(*kbase_gator_hwcnt_init_symbol)(struct kbase_gator_hwcnt_info *); +static void (*kbase_gator_hwcnt_term_symbol)(struct kbase_gator_hwcnt_info *, struct kbase_gator_hwcnt_handles *); + +#else +static struct kbase_device *(*kbase_find_device_symbol)(int); +static struct kbase_context *(*kbase_create_context_symbol)(struct kbase_device *); +static void (*kbase_destroy_context_symbol)(struct kbase_context *); + +#if MALI_DDK_GATOR_API_VERSION == 1 +static void *(*kbase_va_alloc_symbol)(struct kbase_context *, u32); +static void (*kbase_va_free_symbol)(struct kbase_context *, void *); +#elif MALI_DDK_GATOR_API_VERSION == 2 +static void *(*kbase_va_alloc_symbol)(struct kbase_context *, u32, struct kbase_hwc_dma_mapping *); +static void (*kbase_va_free_symbol)(struct kbase_context *, struct kbase_hwc_dma_mapping *); +#endif + +static mali_error (*kbase_instr_hwcnt_enable_symbol)(struct kbase_context *, struct kbase_uk_hwcnt_setup *); +static mali_error (*kbase_instr_hwcnt_disable_symbol)(struct kbase_context *); +static mali_error (*kbase_instr_hwcnt_clear_symbol)(struct kbase_context *); +static mali_error (*kbase_instr_hwcnt_dump_irq_symbol)(struct kbase_context *); +static mali_bool (*kbase_instr_hwcnt_dump_complete_symbol)(struct kbase_context *, mali_bool *); + +static long shader_present_low; +#endif + +/** The interval between reads, in ns. + * + * Earlier we introduced a 'hold off for 1ms after last read' to + * resolve MIDBASE-2178 and MALINE-724. However, the 1ms hold off is + * too long if no context switches occur as there is a race between + * this value and the tick of the read clock in gator which is also + * 1ms. If we 'miss' the current read, the counter values are + * effectively 'spread' over 2ms and the values seen are half what + * they should be (since Streamline averages over sample time). In the + * presence of context switches this spread can vary and markedly + * affect the counters. Currently there is no 'proper' solution to + * this, but empirically we have found that reducing the minimum read + * interval to 950us causes the counts to be much more stable. + */ +static const int READ_INTERVAL_NSEC = 950000; + +#if MALI_DDK_GATOR_API_VERSION != 3 +/* Blocks for HW counters */ +enum { + JM_BLOCK = 0, + TILER_BLOCK, + SHADER_BLOCK, + MMU_BLOCK +}; +#endif + +/* Counters for Mali Midgard or Bifrost: + * + * For HW counters we need strings to create /dev/gator/events files. + * Enums are not needed because the position of the HW name in the array is the same + * of the corresponding value in the received block of memory. + * HW counters are requested by calculating a bitmask, passed then to the driver. + * Every millisecond a HW counters dump is requested, and if the previous has been completed they are read. + */ + +/* Hardware Counters */ +#if MALI_DDK_GATOR_API_VERSION == 3 + +static const char *const *hardware_counter_names; +static int number_of_hardware_counters; + +#else + +static const char *const hardware_counter_names[] = { + /* Job Manager */ + "", + "", + "", + "", + "MESSAGES_SENT", + "MESSAGES_RECEIVED", + "GPU_ACTIVE", /* 6 */ + "IRQ_ACTIVE", + "JS0_JOBS", + "JS0_TASKS", + "JS0_ACTIVE", + "", + "JS0_WAIT_READ", + "JS0_WAIT_ISSUE", + "JS0_WAIT_DEPEND", + "JS0_WAIT_FINISH", + "JS1_JOBS", + "JS1_TASKS", + "JS1_ACTIVE", + "", + "JS1_WAIT_READ", + "JS1_WAIT_ISSUE", + "JS1_WAIT_DEPEND", + "JS1_WAIT_FINISH", + "JS2_JOBS", + "JS2_TASKS", + "JS2_ACTIVE", + "", + "JS2_WAIT_READ", + "JS2_WAIT_ISSUE", + "JS2_WAIT_DEPEND", + "JS2_WAIT_FINISH", + "JS3_JOBS", + "JS3_TASKS", + "JS3_ACTIVE", + "", + "JS3_WAIT_READ", + "JS3_WAIT_ISSUE", + "JS3_WAIT_DEPEND", + "JS3_WAIT_FINISH", + "JS4_JOBS", + "JS4_TASKS", + "JS4_ACTIVE", + "", + "JS4_WAIT_READ", + "JS4_WAIT_ISSUE", + "JS4_WAIT_DEPEND", + "JS4_WAIT_FINISH", + "JS5_JOBS", + "JS5_TASKS", + "JS5_ACTIVE", + "", + "JS5_WAIT_READ", + "JS5_WAIT_ISSUE", + "JS5_WAIT_DEPEND", + "JS5_WAIT_FINISH", + "JS6_JOBS", + "JS6_TASKS", + "JS6_ACTIVE", + "", + "JS6_WAIT_READ", + "JS6_WAIT_ISSUE", + "JS6_WAIT_DEPEND", + "JS6_WAIT_FINISH", + + /*Tiler */ + "", + "", + "", + "JOBS_PROCESSED", + "TRIANGLES", + "QUADS", + "POLYGONS", + "POINTS", + "LINES", + "VCACHE_HIT", + "VCACHE_MISS", + "FRONT_FACING", + "BACK_FACING", + "PRIM_VISIBLE", + "PRIM_CULLED", + "PRIM_CLIPPED", + "LEVEL0", + "LEVEL1", + "LEVEL2", + "LEVEL3", + "LEVEL4", + "LEVEL5", + "LEVEL6", + "LEVEL7", + "COMMAND_1", + "COMMAND_2", + "COMMAND_3", + "COMMAND_4", + "COMMAND_4_7", + "COMMAND_8_15", + "COMMAND_16_63", + "COMMAND_64", + "COMPRESS_IN", + "COMPRESS_OUT", + "COMPRESS_FLUSH", + "TIMESTAMPS", + "PCACHE_HIT", + "PCACHE_MISS", + "PCACHE_LINE", + "PCACHE_STALL", + "WRBUF_HIT", + "WRBUF_MISS", + "WRBUF_LINE", + "WRBUF_PARTIAL", + "WRBUF_STALL", + "ACTIVE", + "LOADING_DESC", + "INDEX_WAIT", + "INDEX_RANGE_WAIT", + "VERTEX_WAIT", + "PCACHE_WAIT", + "WRBUF_WAIT", + "BUS_READ", + "BUS_WRITE", + "", + "", + "", + "", + "", + "UTLB_STALL", + "UTLB_REPLAY_MISS", + "UTLB_REPLAY_FULL", + "UTLB_NEW_MISS", + "UTLB_HIT", + + /* Shader Core */ + "", + "", + "", + "SHADER_CORE_ACTIVE", + "FRAG_ACTIVE", + "FRAG_PRIMATIVES", + "FRAG_PRIMATIVES_DROPPED", + "FRAG_CYCLE_DESC", + "FRAG_CYCLES_PLR", + "FRAG_CYCLES_VERT", + "FRAG_CYCLES_TRISETUP", + "FRAG_CYCLES_RAST", + "FRAG_THREADS", + "FRAG_DUMMY_THREADS", + "FRAG_QUADS_RAST", + "FRAG_QUADS_EZS_TEST", + "FRAG_QUADS_EZS_KILLED", + "FRAG_QUADS_LZS_TEST", + "FRAG_QUADS_LZS_KILLED", + "FRAG_CYCLE_NO_TILE", + "FRAG_NUM_TILES", + "FRAG_TRANS_ELIM", + "COMPUTE_ACTIVE", + "COMPUTE_TASKS", + "COMPUTE_THREADS", + "COMPUTE_CYCLES_DESC", + "TRIPIPE_ACTIVE", + "ARITH_WORDS", + "ARITH_CYCLES_REG", + "ARITH_CYCLES_L0", + "ARITH_FRAG_DEPEND", + "LS_WORDS", + "LS_ISSUES", + "LS_RESTARTS", + "LS_REISSUES_MISS", + "LS_REISSUES_VD", + "LS_REISSUE_ATTRIB_MISS", + "LS_NO_WB", + "TEX_WORDS", + "TEX_BUBBLES", + "TEX_WORDS_L0", + "TEX_WORDS_DESC", + "TEX_THREADS", + "TEX_RECIRC_FMISS", + "TEX_RECIRC_DESC", + "TEX_RECIRC_MULTI", + "TEX_RECIRC_PMISS", + "TEX_RECIRC_CONF", + "LSC_READ_HITS", + "LSC_READ_MISSES", + "LSC_WRITE_HITS", + "LSC_WRITE_MISSES", + "LSC_ATOMIC_HITS", + "LSC_ATOMIC_MISSES", + "LSC_LINE_FETCHES", + "LSC_DIRTY_LINE", + "LSC_SNOOPS", + "AXI_TLB_STALL", + "AXI_TLB_MIESS", + "AXI_TLB_TRANSACTION", + "LS_TLB_MISS", + "LS_TLB_HIT", + "AXI_BEATS_READ", + "AXI_BEATS_WRITTEN", + + /*L2 and MMU */ + "", + "", + "", + "", + "MMU_HIT", + "MMU_NEW_MISS", + "MMU_REPLAY_FULL", + "MMU_REPLAY_MISS", + "MMU_TABLE_WALK", + "", + "", + "", + "", + "", + "", + "", + "UTLB_HIT", + "UTLB_NEW_MISS", + "UTLB_REPLAY_FULL", + "UTLB_REPLAY_MISS", + "UTLB_STALL", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "L2_WRITE_BEATS", + "L2_READ_BEATS", + "L2_ANY_LOOKUP", + "L2_READ_LOOKUP", + "L2_SREAD_LOOKUP", + "L2_READ_REPLAY", + "L2_READ_SNOOP", + "L2_READ_HIT", + "L2_CLEAN_MISS", + "L2_WRITE_LOOKUP", + "L2_SWRITE_LOOKUP", + "L2_WRITE_REPLAY", + "L2_WRITE_SNOOP", + "L2_WRITE_HIT", + "L2_EXT_READ_FULL", + "L2_EXT_READ_HALF", + "L2_EXT_WRITE_FULL", + "L2_EXT_WRITE_HALF", + "L2_EXT_READ", + "L2_EXT_READ_LINE", + "L2_EXT_WRITE", + "L2_EXT_WRITE_LINE", + "L2_EXT_WRITE_SMALL", + "L2_EXT_BARRIER", + "L2_EXT_AR_STALL", + "L2_EXT_R_BUF_FULL", + "L2_EXT_RD_BUF_FULL", + "L2_EXT_R_RAW", + "L2_EXT_W_STALL", + "L2_EXT_W_BUF_FULL", + "L2_EXT_R_W_HAZARD", + "L2_TAG_HAZARD", + "L2_SNOOP_FULL", + "L2_REPLAY_FULL" +}; + +static const int number_of_hardware_counters = ARRAY_SIZE(hardware_counter_names); + +#endif + +// We assume that there will be always not more than 4 blocks of 64 counters in each. +#define GET_HW_BLOCK(c) (((c) >> 6) & 0x3) +#define GET_COUNTER_OFFSET(c) ((c) & 0x3f) + +#if MALI_DDK_GATOR_API_VERSION == 3 +/* Opaque handles for kbase_context and kbase_hwc_dma_mapping */ +static struct kbase_gator_hwcnt_handles *handles; + +/* Information about hardware counters */ +static struct kbase_gator_hwcnt_info *in_out_info; + +#else +/* Memory to dump hardware counters into */ +static void *kernel_dump_buffer; + +#if MALI_DDK_GATOR_API_VERSION == 2 +/* DMA state used to manage lifetime of the buffer */ +struct kbase_hwc_dma_mapping kernel_dump_buffer_handle; +#endif + +/* kbase context and device */ +static struct kbase_context *kbcontext; +static struct kbase_device *kbdevice; + +/* + * The following function has no external prototype in older DDK + * revisions. When the DDK is updated then this should be removed. + */ +struct kbase_device *kbase_find_device(int minor); +#endif + +static volatile bool kbase_device_busy; +static unsigned int num_hardware_counters_enabled; + +/* gatorfs variables for counter enable state */ +static struct mali_counter *counters; + +/* An array used to return the data we recorded as key,value pairs */ +static long long *counter_dump; +static uint64_t last_read_time; + +extern struct mali_counter mali_activity[3]; + +static const char *const mali_activity_names[] = { + "fragment", + "vertex", + "opencl", +}; + +#include "gator_events_mali_midgard.h" + +/** + * Execute symbol_get for all the Mali symbols and check for success. + * @return the number of symbols not loaded. + */ +static int init_symbols(void) +{ + int error_count = 0; +#if MALI_DDK_GATOR_API_VERSION == 3 + SYMBOL_GET(kbase_gator_instr_hwcnt_dump_irq, error_count); + SYMBOL_GET(kbase_gator_instr_hwcnt_dump_complete, error_count); + SYMBOL_GET(kbase_gator_hwcnt_init, error_count); + SYMBOL_GET(kbase_gator_hwcnt_term, error_count); +#else + SYMBOL_GET(kbase_find_device, error_count); + SYMBOL_GET(kbase_create_context, error_count); + SYMBOL_GET(kbase_va_alloc, error_count); + SYMBOL_GET(kbase_instr_hwcnt_enable, error_count); + SYMBOL_GET(kbase_instr_hwcnt_clear, error_count); + SYMBOL_GET(kbase_instr_hwcnt_dump_irq, error_count); + SYMBOL_GET(kbase_instr_hwcnt_dump_complete, error_count); + SYMBOL_GET(kbase_instr_hwcnt_disable, error_count); + SYMBOL_GET(kbase_va_free, error_count); + SYMBOL_GET(kbase_destroy_context, error_count); +#endif + + return error_count; +} + +/** + * Execute symbol_put for all the registered Mali symbols. + */ +static void clean_symbols(void) +{ +#if MALI_DDK_GATOR_API_VERSION == 3 + SYMBOL_CLEANUP(kbase_gator_instr_hwcnt_dump_irq); + SYMBOL_CLEANUP(kbase_gator_instr_hwcnt_dump_complete); + SYMBOL_CLEANUP(kbase_gator_hwcnt_init); + SYMBOL_CLEANUP(kbase_gator_hwcnt_term); +#else + SYMBOL_CLEANUP(kbase_find_device); + SYMBOL_CLEANUP(kbase_create_context); + SYMBOL_CLEANUP(kbase_va_alloc); + SYMBOL_CLEANUP(kbase_instr_hwcnt_enable); + SYMBOL_CLEANUP(kbase_instr_hwcnt_clear); + SYMBOL_CLEANUP(kbase_instr_hwcnt_dump_irq); + SYMBOL_CLEANUP(kbase_instr_hwcnt_dump_complete); + SYMBOL_CLEANUP(kbase_instr_hwcnt_disable); + SYMBOL_CLEANUP(kbase_va_free); + SYMBOL_CLEANUP(kbase_destroy_context); +#endif +} + +static int start(void) +{ +#if MALI_DDK_GATOR_API_VERSION != 3 + struct kbase_uk_hwcnt_setup setup; + unsigned long long shadersPresent = 0; + u16 bitmask[] = { 0, 0, 0, 0 }; + mali_error err; +#endif + int cnt; + + last_read_time = 0; + +#if MALI_DDK_GATOR_API_VERSION == 3 + /* Setup HW counters */ + num_hardware_counters_enabled = 0; + + /* Declare and initialise kbase_gator_hwcnt_info structure */ + in_out_info = kmalloc(sizeof(*in_out_info), GFP_KERNEL); + for (cnt = 0; cnt < ARRAY_SIZE(in_out_info->bitmask); cnt++) + in_out_info->bitmask[cnt] = 0; + + /* Calculate enable bitmasks based on counters_enabled array */ + for (cnt = 0; cnt < number_of_hardware_counters; cnt++) { + if (counters[cnt].enabled) { + int block = GET_HW_BLOCK(cnt); + int enable_bit = GET_COUNTER_OFFSET(cnt) / 4; + + in_out_info->bitmask[block] |= (1 << enable_bit); + pr_debug("gator: %s: hardware counter %s selected [%d]\n", mali_name, hardware_counter_names[cnt], cnt); + num_hardware_counters_enabled++; + } + } + + /* Create a kbase context for HW counters */ + if (num_hardware_counters_enabled > 0) { + if (init_symbols() > 0) { + clean_symbols(); + /* No Mali driver code entrypoints found - not a fault. */ + return 0; + } + + handles = kbase_gator_hwcnt_init_symbol(in_out_info); + + if (handles == NULL) + goto out; + + kbase_device_busy = false; + } + + return 0; +#else + /* Setup HW counters */ + num_hardware_counters_enabled = 0; + + /* Calculate enable bitmasks based on counters_enabled array */ + for (cnt = 0; cnt < number_of_hardware_counters; cnt++) { + const struct mali_counter *counter = &counters[cnt]; + + if (counter->enabled) { + int block = GET_HW_BLOCK(cnt); + int enable_bit = GET_COUNTER_OFFSET(cnt) / 4; + + bitmask[block] |= (1 << enable_bit); + pr_debug("gator: %s: hardware counter %s selected [%d]\n", mali_name, hardware_counter_names[cnt], cnt); + num_hardware_counters_enabled++; + } + } + + /* Create a kbase context for HW counters */ + if (num_hardware_counters_enabled > 0) { + if (init_symbols() > 0) { + clean_symbols(); + /* No Mali driver code entrypoints found - not a fault. */ + return 0; + } + + kbdevice = kbase_find_device_symbol(-1); + + /* If we already got a context, fail */ + if (kbcontext) { + pr_err("gator: Mali-%s: error context already present\n", mali_name); + goto out; + } + + /* kbcontext will only be valid after all the Mali symbols are loaded successfully */ + kbcontext = kbase_create_context_symbol(kbdevice); + if (!kbcontext) { + pr_err("gator: Mali-%s: error creating kbase context\n", mali_name); + goto out; + } + + /* See if we can get the number of shader cores */ + shadersPresent = kbdevice->shader_present_bitmap; + shader_present_low = (unsigned long)shadersPresent; + + /* + * The amount of memory needed to store the dump (bytes) + * DUMP_SIZE = number of core groups + * * number of blocks (always 8 for midgard) + * * number of counters per block (always 64 for midgard) + * * number of bytes per counter (always 4 in midgard) + * For a Mali-Midgard/Bifrost with a single core group = 1 * 8 * 64 * 4 = 2048 + * For a Mali-Midgard/Bifrost with a dual core group = 2 * 8 * 64 * 4 = 4096 + */ +#if MALI_DDK_GATOR_API_VERSION == 1 + kernel_dump_buffer = kbase_va_alloc_symbol(kbcontext, 4096); +#elif MALI_DDK_GATOR_API_VERSION == 2 + kernel_dump_buffer = kbase_va_alloc_symbol(kbcontext, 4096, &kernel_dump_buffer_handle); +#endif + if (!kernel_dump_buffer) { + pr_err("gator: Mali-%s: error trying to allocate va\n", mali_name); + goto destroy_context; + } + + setup.dump_buffer = (uintptr_t)kernel_dump_buffer; + setup.jm_bm = bitmask[JM_BLOCK]; + setup.tiler_bm = bitmask[TILER_BLOCK]; + setup.shader_bm = bitmask[SHADER_BLOCK]; + setup.mmu_l2_bm = bitmask[MMU_BLOCK]; + /* These counters do not exist on Mali-T60x */ + setup.l3_cache_bm = 0; + + /* Use kbase API to enable hardware counters and provide dump buffer */ + err = kbase_instr_hwcnt_enable_symbol(kbcontext, &setup); + if (err != MALI_ERROR_NONE) { + pr_err("gator: Mali-%s: can't setup hardware counters\n", mali_name); + goto free_buffer; + } + pr_debug("gator: Mali-%s: hardware counters enabled\n", mali_name); + kbase_instr_hwcnt_clear_symbol(kbcontext); + pr_debug("gator: Mali-%s: hardware counters cleared\n", mali_name); + + kbase_device_busy = false; + } + + return 0; + +free_buffer: +#if MALI_DDK_GATOR_API_VERSION == 1 + kbase_va_free_symbol(kbcontext, kernel_dump_buffer); +#elif MALI_DDK_GATOR_API_VERSION == 2 + kbase_va_free_symbol(kbcontext, &kernel_dump_buffer_handle); +#endif + +destroy_context: + kbase_destroy_context_symbol(kbcontext); +#endif + +out: + clean_symbols(); + return -1; +} + +static void stop(void) +{ + unsigned int cnt; +#if MALI_DDK_GATOR_API_VERSION == 3 + struct kbase_gator_hwcnt_handles *temp_hand; +#else + struct kbase_context *temp_kbcontext; +#endif + + pr_debug("gator: Mali-%s: stop\n", mali_name); + + /* Set all counters as disabled */ + for (cnt = 0; cnt < number_of_hardware_counters; cnt++) + counters[cnt].enabled = 0; + + /* Destroy the context for HW counters */ +#if MALI_DDK_GATOR_API_VERSION == 3 + if (num_hardware_counters_enabled > 0 && handles != NULL) { + /* + * Set the global variable to NULL before destroying it, because + * other function will check this before using it. + */ + temp_hand = handles; + handles = NULL; + + kbase_gator_hwcnt_term_symbol(in_out_info, temp_hand); + + kfree(in_out_info); + +#else + if (num_hardware_counters_enabled > 0 && kbcontext != NULL) { + /* + * Set the global variable to NULL before destroying it, because + * other function will check this before using it. + */ + temp_kbcontext = kbcontext; + kbcontext = NULL; + + kbase_instr_hwcnt_disable_symbol(temp_kbcontext); + +#if MALI_DDK_GATOR_API_VERSION == 1 + kbase_va_free_symbol(temp_kbcontext, kernel_dump_buffer); +#elif MALI_DDK_GATOR_API_VERSION == 2 + kbase_va_free_symbol(temp_kbcontext, &kernel_dump_buffer_handle); +#endif + + kbase_destroy_context_symbol(temp_kbcontext); +#endif + + pr_debug("gator: Mali-%s: hardware counters stopped\n", mali_name); + + clean_symbols(); + } +} + +static int read_counter(const int cnt, const int len, const struct mali_counter *counter) +{ + const int counter_block = GET_HW_BLOCK(cnt); + const int counter_offset = GET_COUNTER_OFFSET(cnt); + u32 value = 0; + +#if MALI_DDK_GATOR_API_VERSION == 3 + const char *block_base_address = (char *)in_out_info->kernel_dump_buffer; + int i; + int shader_core_count = 0; + int block_type = 0; + + for (i = 0; i < in_out_info->nr_hwc_blocks; i++) + { + block_type = in_out_info->hwc_layout[i]; + + // Being explicit: regardless of what's coming next + // make sure we do skip reserved block types when present + if (block_type != RESERVED_BLOCK) + { + if (counter_block == block_type) + { + value += *((u32 *)(block_base_address + (0x100 * i)) + counter_offset); + if (counter_block == SHADER_BLOCK) + ++shader_core_count; + } + } + } + + if (shader_core_count > 1) + value /= shader_core_count; +#else + const unsigned int vithar_blocks[] = { + 0x700, /* VITHAR_JOB_MANAGER, Block 0 */ + 0x400, /* VITHAR_TILER, Block 1 */ + 0x000, /* VITHAR_SHADER_CORE, Block 2 */ + 0x500 /* VITHAR_MEMORY_SYSTEM, Block 3 */ + }; + const char *block_base_address = (char *)kernel_dump_buffer + vithar_blocks[counter_block]; + + /* If counter belongs to shader block need to take into account all cores */ + if (counter_block == SHADER_BLOCK) { + int i = 0; + int shader_core_count = 0; + + value = 0; + + for (i = 0; i < 4; i++) { + if ((shader_present_low >> i) & 1) { + value += *((u32 *)(block_base_address + (0x100 * i)) + counter_offset); + shader_core_count++; + } + } + + for (i = 0; i < 4; i++) { + if ((shader_present_low >> (i+4)) & 1) { + value += *((u32 *)(block_base_address + (0x100 * i) + 0x800) + counter_offset); + shader_core_count++; + } + } + + /* Need to total by number of cores to produce an average */ + if (shader_core_count != 0) + value /= shader_core_count; + } else { + value = *((u32 *)block_base_address + counter_offset); + } +#endif + + counter_dump[len + 0] = counter->key; + counter_dump[len + 1] = value; + + return 2; +} + +static int read(long long **buffer, bool sched_switch) +{ + int cnt; + int len = 0; + uint32_t success; + + uint64_t curr_time; + + if (!on_primary_core() || sched_switch) + return 0; + + /* + * Report the HW counters + * Only process hardware counters if at least one of the hardware counters is enabled. + */ + if (num_hardware_counters_enabled <= 0) + return 0; + + curr_time = gator_get_time(); + + /* + * Discard reads unless a respectable time has passed. This + * reduces the load on the GPU without sacrificing accuracy on + * the Streamline display. + */ + if (curr_time - last_read_time < READ_INTERVAL_NSEC) + return 0; + +#if MALI_DDK_GATOR_API_VERSION == 3 + if (!handles) + return -1; + + /* Mali symbols can be called safely since a kbcontext is valid */ + if (kbase_gator_instr_hwcnt_dump_complete_symbol(handles, &success)) { +#else + if (!kbcontext) + return -1; + + /* Mali symbols can be called safely since a kbcontext is valid */ + if (kbase_instr_hwcnt_dump_complete_symbol(kbcontext, &success)) { +#endif + kbase_device_busy = false; + + /* + * If last_read_time is zero, then this result is from a previous + * capture or in error. + */ + if (success && last_read_time > 0) { + /* Backdate these events to when they were requested */ + counter_dump[len++] = 0; + counter_dump[len++] = last_read_time; + + /* Cycle through hardware counters and accumulate totals */ + for (cnt = 0; cnt < number_of_hardware_counters; cnt++) { + const struct mali_counter *counter = &counters[cnt]; + + if (counter->enabled) + len += read_counter(cnt, len, counter); + } + + /* Restore the timestamp */ + counter_dump[len++] = 0; + counter_dump[len++] = curr_time; + } + } + + if (!kbase_device_busy) { + kbase_device_busy = true; + last_read_time = curr_time; +#if MALI_DDK_GATOR_API_VERSION == 3 + kbase_gator_instr_hwcnt_dump_irq_symbol(handles); +#else + kbase_instr_hwcnt_dump_irq_symbol(kbcontext); +#endif + } + + /* Update the buffer */ + if (buffer) + *buffer = counter_dump; + + return len; +} + +static int create_files(struct super_block *sb, struct dentry *root) +{ + unsigned int event; + const char* gpu_core_name = NULL; + + // Create the filesystem for activity events + for (event = 0; event < ARRAY_SIZE(mali_activity); event++) { + if (gator_mali_create_file_system(mali_name, mali_activity_names[event], sb, root, &mali_activity[event], NULL) != 0) + return -1; + } + + // Create the filesystem for HW counters events +#if MALI_DDK_GATOR_API_VERSION < 3 + // On older Midgard driver versions we do not have gpu core name added to the counter names, + // so we use Midgard as some form of generic core name as well here. + gpu_core_name = mali_name; +#else + // On newer Midgard and Bifrost drivers, we keep the GPU core name as NULL, because the HW counter names + // are already prefixed with gpu core name (as such returned by the DDK driver). +#endif + for (event = 0; event < number_of_hardware_counters; event++) { + if (gator_mali_create_file_system(gpu_core_name, hardware_counter_names[event], sb, root, &counters[event], NULL) != 0) + return -1; + } + + return 0; +} + +static void shutdown(void) +{ +#if MALI_DDK_GATOR_API_VERSION == 3 + void (*kbase_gator_hwcnt_term_names_symbol)(void) = NULL; + int error_count = 0; +#endif + + kfree(counters); + kfree(counter_dump); + +#if MALI_DDK_GATOR_API_VERSION == 3 + SYMBOL_GET(kbase_gator_hwcnt_term_names, error_count); + + number_of_hardware_counters = -1; + hardware_counter_names = NULL; + if (kbase_gator_hwcnt_term_names_symbol != NULL) { + kbase_gator_hwcnt_term_names_symbol(); + pr_debug("gator: Mali-%s: Released symbols\n", mali_name); + } + + SYMBOL_CLEANUP(kbase_gator_hwcnt_term_names); +#endif +} + +static struct gator_interface gator_events_mali_midgard_interface = { + .name = "mali_hw_counters", + .shutdown = shutdown, + .create_files = create_files, + .start = start, + .stop = stop, + .read64 = read +}; + +// ----------------------------------------------------------------------------------------------- +// Some defines extracted from mali_kbase_gpu_id.h file - to make it available also when +// compiling gator driver against older DDK driver versions that do not have this header. +// ----------------------------------------------------------------------------------------------- +#define GPU_ID_PI_T60X 0x6956 +#define GPU_ID_PI_NEW_FORMAT_START 0x1000 +#define GPU_ID_IS_NEW_FORMAT(product_id) ((product_id) != GPU_ID_PI_T60X && \ + (product_id) >= \ + GPU_ID_PI_NEW_FORMAT_START) + +// Note: what we get in gpu_id field are upper 16 bits from GPU_ID register, therefore: +// - the GPU ARCH_MAJOR value is encoded on bits [15:12] +// - the GPU ARCH_MINOR value is encoded on bits [11:8] +// - the GPU ARCH_REV value is encoded on bits [7:4] +// - the GPU PRODUCT_MAJOR value is encoded on bits [3:0] +#define GPU_ID2_ARCH_MAJOR_SHIFT 12 +// ----------------------------------------------------------------------------------------------- + + +int gator_events_mali_midgard_hw_init(void) +{ + // This is the place when we can safely setup a correct name for the mali architecture + // to make sure software counter names match the XML definitions in the daemon side. + // Depending on the driver interface version, we either know this already + // at the compile time (older interface versions) or + // at the runtime (from MALI_DDK_GATOR_API_VERSION == 3 on). + + static const char* GPU_FAMILY_NAME_MIDGARD = "Midgard"; + int error_count = 0; + +#if MALI_DDK_GATOR_API_VERSION == 3 + static const char* GPU_FAMILY_NAME_BIFROST = "Bifrost"; + struct kbase_gator_hwcnt_info info; + struct kbase_gator_hwcnt_handles * result = NULL; + uint32_t gpu_id = 0; + const char *const *(*kbase_gator_hwcnt_init_names_symbol)(uint32_t *) = NULL; + + // In this case, we have to use DDK driver interface and determine the actual type of the + // GPU architecture at the runtime. That's because the same DDK driver code base supports + // both: Midgards and Bifrosts now. + SYMBOL_GET(kbase_gator_hwcnt_init, error_count); + if (error_count > 0) + { + // Could not find the symbol in the kernel. Can't proceed. + SYMBOL_CLEANUP(kbase_gator_hwcnt_init); + return 1; + } + + // a temporary structure that we'll use to obtain gpu id + result = kbase_gator_hwcnt_init_symbol(&info); + SYMBOL_CLEANUP(kbase_gator_hwcnt_init); + if (result == NULL) + { + // Could not get the info about the GPU. Can't proceed. + return 1; + } + + // now, the info struct has a gpu_id field set to a number, that is architecture-specific + gpu_id = info.gpu_id; + pr_info("gator: Detected GPU ID: %d.\n", gpu_id); + + // Identifying the GPU architecture. Version 6 Bifrost. + if (GPU_ID_IS_NEW_FORMAT(gpu_id) && ((gpu_id >> GPU_ID2_ARCH_MAJOR_SHIFT) & 0xF) >= 6) + { + mali_name = GPU_FAMILY_NAME_BIFROST; + } + else + { + mali_name = GPU_FAMILY_NAME_MIDGARD; + } + + pr_info("gator: Mali-%s architecture detected.\n", mali_name); + +#else + // Older driver API versions were present only on Midgards, so that's straightforward. + mali_name = GPU_FAMILY_NAME_MIDGARD; +#endif + + + pr_debug("gator: Mali-%s: hw_counters init\n", mali_name); + +#if MALI_DDK_GATOR_API_VERSION == 3 + + SYMBOL_GET(kbase_gator_hwcnt_init_names, error_count); + if (error_count > 0) { + SYMBOL_CLEANUP(kbase_gator_hwcnt_init_names); + return 1; + } + + number_of_hardware_counters = -1; + hardware_counter_names = kbase_gator_hwcnt_init_names_symbol(&number_of_hardware_counters); + + SYMBOL_CLEANUP(kbase_gator_hwcnt_init_names); + + if ((hardware_counter_names == NULL) || (number_of_hardware_counters <= 0)) { + pr_err("gator: Mali-%s: Error reading hardware counters names: got %d names\n", mali_name, number_of_hardware_counters); + return -1; + } +#endif + + counters = kmalloc(sizeof(*counters)*number_of_hardware_counters, GFP_KERNEL); + counter_dump = kmalloc(sizeof(*counter_dump)*number_of_hardware_counters*2 + 4, GFP_KERNEL); + + gator_mali_initialise_counters(mali_activity, ARRAY_SIZE(mali_activity)); + gator_mali_initialise_counters(counters, number_of_hardware_counters); + + return gator_events_install(&gator_events_mali_midgard_interface); +} diff --git a/drivers/gator/gator_events_meminfo.c b/drivers/gator/gator_events_meminfo.c new file mode 100644 index 000000000000..a7231c710f61 --- /dev/null +++ b/drivers/gator/gator_events_meminfo.c @@ -0,0 +1,435 @@ +/** + * Copyright (C) Arm Limited 2010-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#include "gator.h" + +#include +#include +#include +#include +#include +#include +#include + +#define USE_THREAD defined(CONFIG_PREEMPT_RT_FULL) + +/* + * Handle rename of global_page_state "c41f012ade0b95b0a6e25c7150673e0554736165 mm: rename global_page_state to global_zone_page_state" + */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0) +#define GLOBAL_ZONE_PAGE_STATE(item) global_page_state(item) +#else +#define GLOBAL_ZONE_PAGE_STATE(item) global_zone_page_state(item) +#endif + +enum { + MEMINFO_MEMFREE, + MEMINFO_MEMUSED, + MEMINFO_BUFFERRAM, + MEMINFO_CACHED, + MEMINFO_SLAB, + MEMINFO_TOTAL, +}; + +enum { + PROC_SIZE, + PROC_SHARE, + PROC_TEXT, + PROC_DATA, + PROC_COUNT, +}; + +static const char * const meminfo_names[] = { + "Linux_meminfo_memfree", + "Linux_meminfo_memused", + "Linux_meminfo_bufferram", + "Linux_meminfo_cached", + "Linux_meminfo_slab", +}; + +static const char * const proc_names[] = { + "Linux_proc_statm_size", + "Linux_proc_statm_share", + "Linux_proc_statm_text", + "Linux_proc_statm_data", +}; + +static bool meminfo_global_enabled; +static ulong meminfo_enabled[MEMINFO_TOTAL]; +static ulong meminfo_keys[MEMINFO_TOTAL]; +static long long meminfo_buffer[2 * (MEMINFO_TOTAL + 2)]; +static int meminfo_length; +static bool new_data_avail; + +static bool proc_global_enabled; +static ulong proc_enabled[PROC_COUNT]; +static ulong proc_keys[PROC_COUNT]; +static DEFINE_PER_CPU(long long, proc_buffer[2 * (PROC_COUNT + 3)]); + +static void do_read(void); + +#if USE_THREAD + +static int gator_meminfo_func(void *data); +static bool gator_meminfo_run; +/* Initialize semaphore unlocked to initialize memory values */ +static DEFINE_SEMAPHORE(gator_meminfo_sem); + +static void notify(void) +{ + up(&gator_meminfo_sem); +} + +#else + +static unsigned int mem_event; +static void wq_sched_handler(struct work_struct *wsptr); +static DECLARE_WORK(work, wq_sched_handler); +static struct timer_list meminfo_wake_up_timer; +static DECLARE_TIMER_HANDLER(meminfo_wake_up_handler); + +static void notify(void) +{ + mem_event++; +} + +#endif + +GATOR_DEFINE_PROBE(mm_page_free, TP_PROTO(struct page *page, unsigned int order)) +{ + notify(); +} + +GATOR_DEFINE_PROBE(mm_page_free_batched, TP_PROTO(struct page *page, int cold)) +{ + notify(); +} + +GATOR_DEFINE_PROBE(mm_page_alloc, TP_PROTO(struct page *page, unsigned int order, gfp_t gfp_flags, int migratetype)) +{ + notify(); +} + +static int gator_events_meminfo_create_files(struct super_block *sb, struct dentry *root) +{ + struct dentry *dir; + int i; + + for (i = 0; i < MEMINFO_TOTAL; i++) { + dir = gatorfs_mkdir(sb, root, meminfo_names[i]); + if (!dir) + return -1; + gatorfs_create_ulong(sb, dir, "enabled", &meminfo_enabled[i]); + gatorfs_create_ro_ulong(sb, dir, "key", &meminfo_keys[i]); + } + + for (i = 0; i < PROC_COUNT; ++i) { + dir = gatorfs_mkdir(sb, root, proc_names[i]); + if (!dir) + return -1; + gatorfs_create_ulong(sb, dir, "enabled", &proc_enabled[i]); + gatorfs_create_ro_ulong(sb, dir, "key", &proc_keys[i]); + } + + return 0; +} + +static int gator_events_meminfo_start(void) +{ + int i; + + new_data_avail = false; + meminfo_global_enabled = 0; + for (i = 0; i < MEMINFO_TOTAL; i++) { + if (meminfo_enabled[i]) { + meminfo_global_enabled = 1; + break; + } + } + + proc_global_enabled = 0; + for (i = 0; i < PROC_COUNT; ++i) { + if (proc_enabled[i]) { + proc_global_enabled = 1; + break; + } + } + if (meminfo_enabled[MEMINFO_MEMUSED]) + proc_global_enabled = 1; + + if (meminfo_global_enabled == 0) + return 0; + + if (GATOR_REGISTER_TRACE(mm_page_free)) + goto mm_page_free_exit; + if (GATOR_REGISTER_TRACE(mm_page_free_batched)) + goto mm_page_free_batched_exit; + if (GATOR_REGISTER_TRACE(mm_page_alloc)) + goto mm_page_alloc_exit; + + do_read(); +#if USE_THREAD + /* Start worker thread */ + gator_meminfo_run = true; + /* Since the mutex starts unlocked, memory values will be initialized */ + if (IS_ERR(kthread_run(gator_meminfo_func, NULL, "gator_meminfo"))) + goto kthread_run_exit; +#else + setup_deferrable_timer_on_stack(&meminfo_wake_up_timer, meminfo_wake_up_handler, 0); +#endif + + return 0; + +#if USE_THREAD +kthread_run_exit: + GATOR_UNREGISTER_TRACE(mm_page_alloc); +#endif +mm_page_alloc_exit: + GATOR_UNREGISTER_TRACE(mm_page_free_batched); +mm_page_free_batched_exit: + GATOR_UNREGISTER_TRACE(mm_page_free); +mm_page_free_exit: + return -1; +} + +static void gator_events_meminfo_stop(void) +{ + if (meminfo_global_enabled) { + GATOR_UNREGISTER_TRACE(mm_page_free); + GATOR_UNREGISTER_TRACE(mm_page_free_batched); + GATOR_UNREGISTER_TRACE(mm_page_alloc); + +#if USE_THREAD + /* Stop worker thread */ + gator_meminfo_run = false; + up(&gator_meminfo_sem); +#else + del_timer_sync(&meminfo_wake_up_timer); +#endif + } +} + +static void do_read(void) +{ + struct sysinfo info; + int i, len; + unsigned long long value; + + meminfo_length = len = 0; + + si_meminfo(&info); + for (i = 0; i < MEMINFO_TOTAL; i++) { + if (meminfo_enabled[i]) { + switch (i) { + case MEMINFO_MEMFREE: + value = info.freeram * PAGE_SIZE; + break; + case MEMINFO_MEMUSED: + /* pid -1 means system wide */ + meminfo_buffer[len++] = 1; + meminfo_buffer[len++] = -1; + /* Emit value */ + meminfo_buffer[len++] = meminfo_keys[MEMINFO_MEMUSED]; + meminfo_buffer[len++] = (info.totalram - info.freeram) * PAGE_SIZE; + /* Clear pid */ + meminfo_buffer[len++] = 1; + meminfo_buffer[len++] = 0; + continue; + case MEMINFO_BUFFERRAM: + value = info.bufferram * PAGE_SIZE; + break; + case MEMINFO_CACHED: + // total_swapcache_pages is not exported so the result is slightly different, but hopefully not too much + value = (GLOBAL_ZONE_PAGE_STATE(NR_FILE_PAGES) /*- total_swapcache_pages()*/ - info.bufferram) * PAGE_SIZE; + break; + case MEMINFO_SLAB: + value = (GLOBAL_ZONE_PAGE_STATE(NR_SLAB_RECLAIMABLE) + GLOBAL_ZONE_PAGE_STATE(NR_SLAB_UNRECLAIMABLE)) * PAGE_SIZE; + break; + default: + value = 0; + break; + } + meminfo_buffer[len++] = meminfo_keys[i]; + meminfo_buffer[len++] = value; + } + } + + meminfo_length = len; + new_data_avail = true; +} + +#if USE_THREAD + +static int gator_meminfo_func(void *data) +{ + for (;;) { + if (down_killable(&gator_meminfo_sem)) + break; + + /* Eat up any pending events */ + while (!down_trylock(&gator_meminfo_sem)) + ; + + if (!gator_meminfo_run) + break; + + do_read(); + } + + return 0; +} + +#else + +/* Must be run in process context as the kernel function si_meminfo() can sleep */ +static void wq_sched_handler(struct work_struct *wsptr) +{ + do_read(); +} + +static DECLARE_TIMER_HANDLER(meminfo_wake_up_handler) +{ + /* had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater */ + schedule_work(&work); +} + +#endif + +static int gator_events_meminfo_read(long long **buffer, bool sched_switch) +{ +#if !USE_THREAD + static unsigned int last_mem_event; +#endif + + if (!on_primary_core() || !meminfo_global_enabled) + return 0; + +#if !USE_THREAD + if (last_mem_event != mem_event) { + last_mem_event = mem_event; + mod_timer(&meminfo_wake_up_timer, jiffies + 1); + } +#endif + + if (!new_data_avail) + return 0; + + new_data_avail = false; + + if (buffer) + *buffer = meminfo_buffer; + + return meminfo_length; +} + +static int gator_events_meminfo_read_proc(long long **buffer, struct task_struct *task) +{ + struct mm_struct *mm; + u64 share = 0; + int i; + long long value; + int len = 0; + int cpu = get_physical_cpu(); + long long *buf = per_cpu(proc_buffer, cpu); + + if (!proc_global_enabled) + return 0; + + /* Collect the memory stats of the process instead of the thread */ + if (task->group_leader != NULL) + task = task->group_leader; + + /* get_task_mm/mmput is not needed in this context because the task and it's mm are required as part of the sched_switch */ + mm = task->mm; + if (mm == NULL) + return 0; + + /* Derived from task_statm in fs/proc/task_mmu.c */ + if (meminfo_enabled[MEMINFO_MEMUSED] || proc_enabled[PROC_SHARE]) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0) + share = get_mm_counter(mm, MM_FILEPAGES); +#else + share = get_mm_counter(mm, MM_FILEPAGES) + get_mm_counter(mm, MM_SHMEMPAGES); +#endif + } + + /* key of 1 indicates a pid */ + buf[len++] = 1; + buf[len++] = task->pid; + + for (i = 0; i < PROC_COUNT; ++i) { + if (proc_enabled[i]) { + switch (i) { + case PROC_SIZE: + value = mm->total_vm; + break; + case PROC_SHARE: + value = share; + break; + case PROC_TEXT: + value = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) >> PAGE_SHIFT; + break; + case PROC_DATA: +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0) + value = mm->total_vm - mm->shared_vm; +#else + value = mm->total_vm - mm->stack_vm; +#endif + break; + } + + buf[len++] = proc_keys[i]; + buf[len++] = value * PAGE_SIZE; + } + } + + if (meminfo_enabled[MEMINFO_MEMUSED]) { + value = share + get_mm_counter(mm, MM_ANONPAGES); + /* Send resident for this pid */ + buf[len++] = meminfo_keys[MEMINFO_MEMUSED]; + buf[len++] = value * PAGE_SIZE; + } + + /* Clear pid */ + buf[len++] = 1; + buf[len++] = 0; + + if (buffer) + *buffer = buf; + + return len; +} + +static struct gator_interface gator_events_meminfo_interface = { + .name = "meminfo", + .create_files = gator_events_meminfo_create_files, + .start = gator_events_meminfo_start, + .stop = gator_events_meminfo_stop, + .read64 = gator_events_meminfo_read, + .read_proc = gator_events_meminfo_read_proc, +}; + +int gator_events_meminfo_init(void) +{ + int i; + + meminfo_global_enabled = 0; + for (i = 0; i < MEMINFO_TOTAL; i++) { + meminfo_enabled[i] = 0; + meminfo_keys[i] = gator_events_get_key(); + } + + proc_global_enabled = 0; + for (i = 0; i < PROC_COUNT; ++i) { + proc_enabled[i] = 0; + proc_keys[i] = gator_events_get_key(); + } + + return gator_events_install(&gator_events_meminfo_interface); +} diff --git a/drivers/gator/gator_events_mmapped.c b/drivers/gator/gator_events_mmapped.c new file mode 100644 index 000000000000..82f06af1e6f1 --- /dev/null +++ b/drivers/gator/gator_events_mmapped.c @@ -0,0 +1,210 @@ +/* + * Example events provider + * + * Copyright (C) Arm Limited 2010-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * Similar entries to those below must be present in the events.xml file. + * To add them to the events.xml, create an events-mmap.xml with the + * following contents and rebuild gatord: + * + * + * + * + * + * + * + * When adding custom events, be sure to do the following: + * - add any needed .c files to the gator driver Makefile + * - call gator_events_install in the events init function + * - add the init function to GATOR_EVENTS_LIST in gator_main.c + * - add a new events-*.xml file to the gator daemon and rebuild + * + * Troubleshooting: + * - verify the new events are part of events.xml, which is created when building the daemon + * - verify the new events exist at /dev/gator/events/ once gatord is launched + * - verify the counter name in the XML matches the name at /dev/gator/events + */ + +#include +#include +#include + +#include "gator.h" + +#define MMAPPED_COUNTERS_NUM 3 + +static int mmapped_global_enabled; + +static struct { + unsigned long enabled; + unsigned long key; +} mmapped_counters[MMAPPED_COUNTERS_NUM]; + +static int mmapped_buffer[MMAPPED_COUNTERS_NUM * 2]; + +static s64 prev_time; + +/* Adds mmapped_cntX directories and enabled, event, and key files to /dev/gator/events */ +static int gator_events_mmapped_create_files(struct super_block *sb, + struct dentry *root) +{ + int i; + + for (i = 0; i < MMAPPED_COUNTERS_NUM; i++) { + char buf[16]; + struct dentry *dir; + + snprintf(buf, sizeof(buf), "mmapped_cnt%d", i); + dir = gatorfs_mkdir(sb, root, buf); + if (WARN_ON(!dir)) + return -1; + gatorfs_create_ulong(sb, dir, "enabled", + &mmapped_counters[i].enabled); + gatorfs_create_ro_ulong(sb, dir, "key", + &mmapped_counters[i].key); + } + + return 0; +} + +static int gator_events_mmapped_start(void) +{ + int i; + struct timespec ts; + + getnstimeofday(&ts); + prev_time = timespec_to_ns(&ts); + + mmapped_global_enabled = 0; + for (i = 0; i < MMAPPED_COUNTERS_NUM; i++) { + if (mmapped_counters[i].enabled) { + mmapped_global_enabled = 1; + break; + } + } + + return 0; +} + +static void gator_events_mmapped_stop(void) +{ +} + +/* This function "simulates" counters, generating values of fancy + * functions like sine or triangle... */ +static int mmapped_simulate(int counter, int delta_in_us) +{ + int result = 0; + + switch (counter) { + case 0: /* sort-of-sine */ + { + static int t; + int x; + + t += delta_in_us; + if (t > 2048000) + t = 0; + + if (t % 1024000 < 512000) + x = 512000 - (t % 512000); + else + x = t % 512000; + + result = 32 * x / 512000; + result = result * result; + + if (t < 1024000) + result = 1922 - result; + } + break; + case 1: /* triangle */ + { + static int v, d = 1; + + v = v + d * delta_in_us; + if (v < 0) { + v = 0; + d = 1; + } else if (v > 1000000) { + v = 1000000; + d = -1; + } + + result = v; + } + break; + case 2: /* PWM signal */ + { + static int dc, x, t; + + t += delta_in_us; + if (t > 1000000) + t = 0; + if (x / 1000000 != (x + delta_in_us) / 1000000) + dc = (dc + 100000) % 1000000; + x += delta_in_us; + + result = t < dc ? 0 : 10; + } + break; + } + + return result; +} + +static int gator_events_mmapped_read(int **buffer, bool sched_switch) +{ + int i; + int len = 0; + int delta_in_us; + struct timespec ts; + s64 time; + + /* System wide counters - read from one core only */ + if (!on_primary_core() || !mmapped_global_enabled) + return 0; + + getnstimeofday(&ts); + time = timespec_to_ns(&ts); + delta_in_us = (int)(time - prev_time) / 1000; + prev_time = time; + + for (i = 0; i < MMAPPED_COUNTERS_NUM; i++) { + if (mmapped_counters[i].enabled) { + mmapped_buffer[len++] = mmapped_counters[i].key; + mmapped_buffer[len++] = + mmapped_simulate(i, delta_in_us); + } + } + + if (buffer) + *buffer = mmapped_buffer; + + return len; +} + +static struct gator_interface gator_events_mmapped_interface = { + .name = "mmapped", + .create_files = gator_events_mmapped_create_files, + .start = gator_events_mmapped_start, + .stop = gator_events_mmapped_stop, + .read = gator_events_mmapped_read, +}; + +/* Must not be static! */ +int gator_events_mmapped_init(void) +{ + int i; + + for (i = 0; i < MMAPPED_COUNTERS_NUM; i++) { + mmapped_counters[i].enabled = 0; + mmapped_counters[i].key = gator_events_get_key(); + } + + return gator_events_install(&gator_events_mmapped_interface); +} diff --git a/drivers/gator/gator_events_net.c b/drivers/gator/gator_events_net.c new file mode 100644 index 000000000000..79afc5f0dcb8 --- /dev/null +++ b/drivers/gator/gator_events_net.c @@ -0,0 +1,165 @@ +/** + * Copyright (C) Arm Limited 2010-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#include "gator.h" +#include +#include + +#define NETRX 0 +#define NETTX 1 +#define TOTALNET 2 + +static ulong netrx_enabled; +static ulong nettx_enabled; +static ulong netrx_key; +static ulong nettx_key; +static int rx_total, tx_total; +static ulong netPrev[TOTALNET]; +static int netGet[TOTALNET * 4]; + +static struct timer_list net_wake_up_timer; + +/* Must be run in process context as the kernel function dev_get_stats() can sleep */ +static void get_network_stats(struct work_struct *wsptr) +{ + int rx = 0, tx = 0; + struct net_device *dev; + + for_each_netdev(&init_net, dev) { + struct rtnl_link_stats64 temp; + const struct rtnl_link_stats64 *stats = dev_get_stats(dev, &temp); + rx += stats->rx_bytes; + tx += stats->tx_bytes; + } + rx_total = rx; + tx_total = tx; +} + +static DECLARE_WORK(wq_get_stats, get_network_stats); + +static DECLARE_TIMER_HANDLER(net_wake_up_handler) +{ + /* had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater */ + schedule_work(&wq_get_stats); +} + +static void calculate_delta(int *rx, int *tx) +{ + int rx_calc, tx_calc; + + rx_calc = (int)(rx_total - netPrev[NETRX]); + if (rx_calc < 0) + rx_calc = 0; + netPrev[NETRX] += rx_calc; + + tx_calc = (int)(tx_total - netPrev[NETTX]); + if (tx_calc < 0) + tx_calc = 0; + netPrev[NETTX] += tx_calc; + + *rx = rx_calc; + *tx = tx_calc; +} + +static int gator_events_net_create_files(struct super_block *sb, struct dentry *root) +{ + /* Network counters are not currently supported in RT-Preempt full because mod_timer is used */ +#ifndef CONFIG_PREEMPT_RT_FULL + struct dentry *dir; + + dir = gatorfs_mkdir(sb, root, "Linux_net_rx"); + if (!dir) + return -1; + gatorfs_create_ulong(sb, dir, "enabled", &netrx_enabled); + gatorfs_create_ro_ulong(sb, dir, "key", &netrx_key); + + dir = gatorfs_mkdir(sb, root, "Linux_net_tx"); + if (!dir) + return -1; + gatorfs_create_ulong(sb, dir, "enabled", &nettx_enabled); + gatorfs_create_ro_ulong(sb, dir, "key", &nettx_key); +#endif + + return 0; +} + +static int gator_events_net_start(void) +{ + get_network_stats(NULL); + netPrev[NETRX] = rx_total; + netPrev[NETTX] = tx_total; + setup_deferrable_timer_on_stack(&net_wake_up_timer, net_wake_up_handler, 0); + return 0; +} + +static void gator_events_net_stop(void) +{ + del_timer_sync(&net_wake_up_timer); + netrx_enabled = 0; + nettx_enabled = 0; +} + +static int gator_events_net_read(int **buffer, bool sched_switch) +{ + int len, rx_delta, tx_delta; + static int last_rx_delta, last_tx_delta; + + if (!on_primary_core()) + return 0; + + if (!netrx_enabled && !nettx_enabled) + return 0; + + mod_timer(&net_wake_up_timer, jiffies + 1); + + calculate_delta(&rx_delta, &tx_delta); + + len = 0; + if (netrx_enabled && last_rx_delta != rx_delta) { + last_rx_delta = rx_delta; + netGet[len++] = netrx_key; + /* indicates to Streamline that rx_delta bytes were transmitted now, not since the last message */ + netGet[len++] = 0; + netGet[len++] = netrx_key; + netGet[len++] = rx_delta; + } + + if (nettx_enabled && last_tx_delta != tx_delta) { + last_tx_delta = tx_delta; + netGet[len++] = nettx_key; + /* indicates to Streamline that tx_delta bytes were transmitted now, not since the last message */ + netGet[len++] = 0; + netGet[len++] = nettx_key; + netGet[len++] = tx_delta; + } + + if (buffer) + *buffer = netGet; + + return len; +} + +static struct gator_interface gator_events_net_interface = { + .name = "net", + .create_files = gator_events_net_create_files, + .start = gator_events_net_start, + .stop = gator_events_net_stop, + .read = gator_events_net_read, +}; + +int gator_events_net_init(void) +{ + netrx_key = gator_events_get_key(); + nettx_key = gator_events_get_key(); + + netrx_enabled = 0; + nettx_enabled = 0; + + return gator_events_install(&gator_events_net_interface); +} diff --git a/drivers/gator/gator_events_perf_pmu.c b/drivers/gator/gator_events_perf_pmu.c new file mode 100644 index 000000000000..826d622108e6 --- /dev/null +++ b/drivers/gator/gator_events_perf_pmu.c @@ -0,0 +1,581 @@ +/** + * Copyright (C) Arm Limited 2010-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include "gator.h" + +/* gator_events_armvX.c is used for Linux 2.6.x */ +#if GATOR_PERF_PMU_SUPPORT + +#include +#include +#include + +/* Maximum number of per-core counters - currently reserves enough space for two full hardware PMUs for big.LITTLE */ +#define CNTMAX 16 +/* Maximum number of uncore counters */ +#define UCCNT 32 + +/* Default to 0 if unable to probe the revision which was the previous behavior */ +#define DEFAULT_CCI_REVISION 0 + +/* A gator_attr is needed for every counter */ +struct gator_attr { + /* Set once in gator_events_perf_pmu_*_init - the name of the event in the gatorfs */ + char name[40]; + /* Exposed in gatorfs - set by gatord to enable this counter */ + unsigned long enabled; + /* Set once in gator_events_perf_pmu_*_init - the perf type to use, see perf_type_id in the perf_event.h header file. */ + unsigned long type; + /* Exposed in gatorfs - set by gatord to select the event to collect */ + unsigned long event; + /* Exposed in gatorfs - set by gatord with the sample period to use and enable EBS for this counter */ + unsigned long count; + /* Exposed as read only in gatorfs - set once in __attr_init as the key to use in the APC data */ + unsigned long key; + /* only one of the two may ever be set */ + /* The gator_cpu object that it belongs to */ + const struct gator_cpu * gator_cpu; + /* The uncore_pmu object that it belongs to */ + const struct uncore_pmu * uncore_pmu; +}; + +/* Per-core counter attributes */ +static struct gator_attr attrs[CNTMAX]; +/* Number of initialized per-core counters */ +static int attr_count; +/* Uncore counter attributes */ +static struct gator_attr uc_attrs[UCCNT]; +/* Number of initialized uncore counters */ +static int uc_attr_count; +/* Mapping from CPU to gator_cpu object */ +static const struct gator_cpu * gator_cpus_per_core[ARRAY_SIZE(gator_cpuids)]; + +struct gator_event { + uint32_t curr; + uint32_t prev; + uint32_t prev_delta; + bool zero; + struct perf_event *pevent; + struct perf_event_attr *pevent_attr; +}; + +static DEFINE_PER_CPU(struct gator_event[CNTMAX], events); +static struct gator_event uc_events[UCCNT]; +static DEFINE_PER_CPU(int[(CNTMAX + UCCNT)*2], perf_cnt); + +static void gator_events_perf_pmu_stop(void); + +static int __create_files(struct super_block *sb, struct dentry *root, struct gator_attr *const attr) +{ + struct dentry *dir; + + if (attr->name[0] == '\0') + return 0; + dir = gatorfs_mkdir(sb, root, attr->name); + if (!dir) + return -1; + gatorfs_create_ulong(sb, dir, "enabled", &attr->enabled); + gatorfs_create_ulong(sb, dir, "count", &attr->count); + gatorfs_create_ro_ulong(sb, dir, "key", &attr->key); + gatorfs_create_ulong(sb, dir, "event", &attr->event); + + return 0; +} + +static int gator_events_perf_pmu_create_files(struct super_block *sb, struct dentry *root) +{ + int cnt; + + for (cnt = 0; cnt < attr_count; cnt++) { + if (__create_files(sb, root, &attrs[cnt]) != 0) + return -1; + } + + for (cnt = 0; cnt < uc_attr_count; cnt++) { + if (__create_files(sb, root, &uc_attrs[cnt]) != 0) + return -1; + } + + return 0; +} + +static void ebs_overflow_handler(struct perf_event *event, struct perf_sample_data *data, struct pt_regs *regs) +{ + gator_backtrace_handler(regs); +} + +static void dummy_handler(struct perf_event *event, struct perf_sample_data *data, struct pt_regs *regs) +{ + /* Required as perf_event_create_kernel_counter() requires an overflow handler, even though all we do is poll */ +} + +static int gator_events_perf_pmu_read(int **buffer, bool sched_switch); + +static int gator_events_perf_pmu_online(int **buffer, bool migrate) +{ + return gator_events_perf_pmu_read(buffer, false); +} + +static void __online_dispatch(int cpu, bool migrate, struct gator_attr *const attr, struct gator_event *const event) +{ + const char * const cpu_core_name = (gator_cpus_per_core[cpu] != NULL ? gator_cpus_per_core[cpu]->core_name : "Unknown"); + const u32 cpu_cpuid = (gator_cpus_per_core[cpu] != NULL ? gator_cpus_per_core[cpu]->cpuid : gator_cpuids[cpu]); + + perf_overflow_handler_t handler; + struct perf_event *pevent; + + event->zero = true; + + if (event->pevent != NULL || event->pevent_attr == NULL || migrate) + return; + + if ((gator_cpus_per_core[cpu] == NULL) || ((gator_cpus_per_core[cpu] != attr->gator_cpu) && (attr->gator_cpu != NULL))) { + pr_debug("gator: Counter %s does not apply to core %i (Core: %s 0x%x, Counter: %s 0x%lx)\n", + attr->name, cpu, cpu_core_name, cpu_cpuid, attr->gator_cpu->core_name, attr->gator_cpu->cpuid); + return; + } + else { + pr_debug("gator: Counter %s applies to core %i (%s 0x%x)\n", attr->name, cpu, cpu_core_name, cpu_cpuid); + } + + if (attr->count > 0) + handler = ebs_overflow_handler; + else + handler = dummy_handler; + + pevent = perf_event_create_kernel_counter(event->pevent_attr, cpu, NULL, handler, NULL); + if (IS_ERR(pevent)) { + pr_err("gator: unable to online a counter on cpu %d\n", cpu); + return; + } + + if (pevent->state != PERF_EVENT_STATE_ACTIVE) { + pr_err("gator: inactive counter on cpu %d\n", cpu); + perf_event_release_kernel(pevent); + return; + } + + event->pevent = pevent; +} + +#define GATOR_IF_UNCORE_CPUMASK(uncore_pmu, cpu, cpumask) \ + (cpumask) = (struct cpumask *) GATOR_ATOMIC_READ(&((uncore_pmu)->cpumask_atomic)); \ + if (((!(cpumask)) && ((cpu) == 0)) || ((cpumask) && cpumask_test_cpu((cpu), (cpumask)))) + +static void gator_events_perf_pmu_online_dispatch(int cpu, bool migrate) +{ + int cnt; + + cpu = pcpu_to_lcpu(cpu); + + for (cnt = 0; cnt < attr_count; cnt++) { + __online_dispatch(cpu, migrate, &attrs[cnt], &per_cpu(events, cpu)[cnt]); + } + + for (cnt = 0; cnt < uc_attr_count; cnt++) { + struct cpumask * cpumask; + GATOR_IF_UNCORE_CPUMASK(uc_attrs[cnt].uncore_pmu, cpu, cpumask) { + __online_dispatch(cpu, migrate, &uc_attrs[cnt], &uc_events[cnt]); + } + } +} + +static void __offline_dispatch(int cpu, struct gator_event *const event) +{ + struct perf_event *pe = NULL; + + if (event->pevent) { + pe = event->pevent; + event->pevent = NULL; + } + + if (pe) + perf_event_release_kernel(pe); +} + +static void gator_events_perf_pmu_offline_dispatch(int cpu, bool migrate) +{ + int cnt; + + if (migrate) + return; + cpu = pcpu_to_lcpu(cpu); + + for (cnt = 0; cnt < attr_count; cnt++) + __offline_dispatch(cpu, &per_cpu(events, cpu)[cnt]); + + for (cnt = 0; cnt < uc_attr_count; cnt++) { + struct cpumask * cpumask; + GATOR_IF_UNCORE_CPUMASK(uc_attrs[cnt].uncore_pmu, cpu, cpumask) { + __offline_dispatch(cpu, &uc_events[cnt]); + } + } +} + +static int __check_ebs(struct gator_attr *const attr) +{ + if (attr->count > 0) { + if (!event_based_sampling) { + event_based_sampling = true; + } else { + pr_warning("gator: Only one ebs counter is allowed\n"); + return -1; + } + } + + return 0; +} + +static int __start(struct gator_attr *const attr, struct gator_event *const event) +{ + u32 size = sizeof(struct perf_event_attr); + + event->pevent = NULL; + /* Skip disabled counters */ + if (!attr->enabled) + return 0; + + event->prev = 0; + event->curr = 0; + event->prev_delta = 0; + event->pevent_attr = kmalloc(size, GFP_KERNEL); + if (!event->pevent_attr) { + gator_events_perf_pmu_stop(); + return -1; + } + + memset(event->pevent_attr, 0, size); + event->pevent_attr->type = attr->type; + event->pevent_attr->size = size; + event->pevent_attr->config = attr->event; + event->pevent_attr->sample_period = attr->count; + event->pevent_attr->pinned = 1; + + return 0; +} + +static int gator_events_perf_pmu_start(void) +{ + int cnt, cpu; + + event_based_sampling = false; + for (cnt = 0; cnt < attr_count; cnt++) { + if (__check_ebs(&attrs[cnt]) != 0) + return -1; + } + + for (cnt = 0; cnt < uc_attr_count; cnt++) { + if (__check_ebs(&uc_attrs[cnt]) != 0) + return -1; + } + + for_each_present_cpu(cpu) { + for (cnt = 0; cnt < attr_count; cnt++) { + if (__start(&attrs[cnt], &per_cpu(events, cpu)[cnt]) != 0) + return -1; + } + } + + for (cnt = 0; cnt < uc_attr_count; cnt++) { + if (__start(&uc_attrs[cnt], &uc_events[cnt]) != 0) + return -1; + } + + return 0; +} + +static void __event_stop(struct gator_event *const event) +{ + kfree(event->pevent_attr); + event->pevent_attr = NULL; +} + +static void __attr_stop(struct gator_attr *const attr) +{ + attr->enabled = 0; + attr->event = 0; + attr->count = 0; +} + +static void gator_events_perf_pmu_stop(void) +{ + unsigned int cnt, cpu; + + for_each_present_cpu(cpu) { + for (cnt = 0; cnt < attr_count; cnt++) + __event_stop(&per_cpu(events, cpu)[cnt]); + } + + for (cnt = 0; cnt < uc_attr_count; cnt++) + __event_stop(&uc_events[cnt]); + + for (cnt = 0; cnt < attr_count; cnt++) + __attr_stop(&attrs[cnt]); + + for (cnt = 0; cnt < uc_attr_count; cnt++) + __attr_stop(&uc_attrs[cnt]); +} + +static void __read(int *const len, int cpu, struct gator_attr *const attr, struct gator_event *const event) +{ + uint32_t delta; + struct perf_event *const ev = event->pevent; + + if (ev != NULL && ev->state == PERF_EVENT_STATE_ACTIVE) { + /* After creating the perf counter in __online_dispatch, there + * is a race condition between gator_events_perf_pmu_online and + * gator_events_perf_pmu_read. So have + * gator_events_perf_pmu_online call gator_events_perf_pmu_read + * and in __read check to see if it's the first call after + * __online_dispatch and if so, run the online code. + */ + if (event->zero) { + ev->pmu->read(ev); + event->prev = event->curr = local64_read(&ev->count); + event->prev_delta = 0; + per_cpu(perf_cnt, cpu)[(*len)++] = attr->key; + per_cpu(perf_cnt, cpu)[(*len)++] = 0; + event->zero = false; + } else { + ev->pmu->read(ev); + event->curr = local64_read(&ev->count); + delta = event->curr - event->prev; + if (delta != 0 || delta != event->prev_delta) { + event->prev_delta = delta; + event->prev = event->curr; + per_cpu(perf_cnt, cpu)[(*len)++] = attr->key; + per_cpu(perf_cnt, cpu)[(*len)++] = delta; + } + } + } +} + +static int gator_events_perf_pmu_read(int **buffer, bool sched_switch) +{ + int cnt, len = 0; + const int cpu = get_logical_cpu(); + + for (cnt = 0; cnt < attr_count; cnt++) + __read(&len, cpu, &attrs[cnt], &per_cpu(events, cpu)[cnt]); + + for (cnt = 0; cnt < uc_attr_count; cnt++) { + struct cpumask * cpumask; + GATOR_IF_UNCORE_CPUMASK(uc_attrs[cnt].uncore_pmu, cpu, cpumask) { + __read(&len, cpu, &uc_attrs[cnt], &uc_events[cnt]); + } + } + + if (buffer) + *buffer = per_cpu(perf_cnt, cpu); + + return len; +} + +static struct gator_interface gator_events_perf_pmu_interface = { + .name = "perf_pmu", + .start = gator_events_perf_pmu_start, + .stop = gator_events_perf_pmu_stop, + .online = gator_events_perf_pmu_online, + .online_dispatch = gator_events_perf_pmu_online_dispatch, + .offline_dispatch = gator_events_perf_pmu_offline_dispatch, + .read = gator_events_perf_pmu_read, +}; + +static void __attr_init(struct gator_attr *const attr) +{ + attr->name[0] = '\0'; + attr->enabled = 0; + attr->type = 0; + attr->event = 0; + attr->count = 0; + attr->key = gator_events_get_key(); + attr->gator_cpu = NULL; + attr->uncore_pmu = NULL; +} + +static void gator_events_perf_pmu_uncore_init(const struct uncore_pmu *const uncore_pmu, const int type) +{ + int cnt; + + if (uncore_pmu->has_cycles_counter) { + if (uc_attr_count < ARRAY_SIZE(uc_attrs)) { + snprintf(uc_attrs[uc_attr_count].name, sizeof(uc_attrs[uc_attr_count].name), "%s_ccnt", uncore_pmu->core_name); + uc_attrs[uc_attr_count].type = type; + uc_attrs[uc_attr_count].gator_cpu = NULL; + uc_attrs[uc_attr_count].uncore_pmu = uncore_pmu; + } + ++uc_attr_count; + } + + for (cnt = 0; cnt < uncore_pmu->pmnc_counters; ++cnt, ++uc_attr_count) { + struct gator_attr *const attr = &uc_attrs[uc_attr_count]; + + if (uc_attr_count < ARRAY_SIZE(uc_attrs)) { + snprintf(attr->name, sizeof(attr->name), "%s_cnt%d", uncore_pmu->core_name, cnt); + attr->type = type; + attr->gator_cpu = NULL; + attr->uncore_pmu = uncore_pmu; + } + } +} + +static void gator_events_perf_pmu_cpu_init(const struct gator_cpu *const gator_cpu, const int type) +{ + int cnt; + + if (gator_cluster_count < ARRAY_SIZE(gator_clusters)) { + gator_clusters[gator_cluster_count++] = gator_cpu; + + if (attr_count < ARRAY_SIZE(attrs)) { + snprintf(attrs[attr_count].name, sizeof(attrs[attr_count].name), "%s_ccnt", gator_cpu->pmnc_name); + attrs[attr_count].type = type; + attrs[attr_count].gator_cpu = gator_cpu; + attrs[uc_attr_count].uncore_pmu = NULL; + } + ++attr_count; + + for (cnt = 0; cnt < gator_cpu->pmnc_counters; ++cnt, ++attr_count) { + struct gator_attr *const attr = &attrs[attr_count]; + + if (attr_count < ARRAY_SIZE(attrs)) { + snprintf(attr->name, sizeof(attr->name), "%s_cnt%d", gator_cpu->pmnc_name, cnt); + attr->type = type; + attr->gator_cpu = gator_cpu; + attr->uncore_pmu = NULL; + } + } + } +} + +static int gator_events_perf_pmu_reread(void) +{ + struct perf_event_attr pea; + struct perf_event *pe; + const struct gator_cpu *gator_cpu; + const struct uncore_pmu *uncore_pmu; + int type; + int cpu; + int cnt; + + for (cnt = 0; cnt < ARRAY_SIZE(attrs); cnt++) + __attr_init(&attrs[cnt]); + for (cnt = 0; cnt < ARRAY_SIZE(uc_attrs); cnt++) + __attr_init(&uc_attrs[cnt]); + for (cnt = 0; cnt < ARRAY_SIZE(gator_cpus_per_core); cnt++) + gator_cpus_per_core[cnt] = NULL; + + memset(&pea, 0, sizeof(pea)); + pea.size = sizeof(pea); + pea.config = 0xFF; + attr_count = 0; + uc_attr_count = 0; + for (type = PERF_TYPE_MAX; type < 0x20; ++type) { + pea.type = type; + + /* A particular PMU may work on some but not all cores, so try on each core */ + pe = NULL; + for_each_present_cpu(cpu) { + pe = perf_event_create_kernel_counter(&pea, cpu, NULL, dummy_handler, NULL); + if (!IS_ERR(pe)) + break; + } + /* Assume that valid PMUs are contiguous */ + if (IS_ERR(pe)) { + pea.config = 0xff00; + pe = perf_event_create_kernel_counter(&pea, 0, NULL, dummy_handler, NULL); + if (IS_ERR(pe)) + break; + } + + if (pe->pmu != NULL && type == pe->pmu->type) { + pr_notice("gator: perf pmu: %s\n", pe->pmu->name); + if ((uncore_pmu = gator_find_uncore_pmu(pe->pmu->name)) != NULL) { + pr_notice("gator: Adding uncore counters for %s (%s) with type %i\n", uncore_pmu->core_name, uncore_pmu->pmnc_name, type); + gator_events_perf_pmu_uncore_init(uncore_pmu, type); + } else if ((gator_cpu = gator_find_cpu_by_pmu_name(pe->pmu->name)) != NULL) { + pr_notice("gator: Adding cpu counters for %s (%s) with type %i\n", gator_cpu->core_name, gator_cpu->pmnc_name, type); + gator_events_perf_pmu_cpu_init(gator_cpu, type); + } + /* Initialize gator_attrs for dynamic PMUs here */ + } + + perf_event_release_kernel(pe); + } + + /* Now use CPUID to create CPU->PMU mapping, and add any missing PMUs using PERF_TYPE_RAW */ + for (cpu = 0; cpu < ARRAY_SIZE(gator_cpuids); ++cpu) { + if (gator_cpuids[cpu] != ((u32) -1)) { + bool found_cpu = false; + const struct gator_cpu *gator_cpu = gator_find_cpu_by_cpuid(gator_cpuids[cpu]); + +#if defined(__arm__) || defined(__aarch64__) + if (gator_cpu == NULL) { + pr_err("gator: This CPU is not recognized, using the Arm architected counters\n"); + gator_cpu = &gator_pmu_other; + } +#else + if (gator_cpu == NULL) { + pr_err("gator: This CPU is not recognized\n"); + return -1; + } +#endif + for (cnt = 0; cnt < gator_cluster_count; ++cnt) { + if (gator_clusters[cnt] == gator_cpu) { + found_cpu = true; + break; + } + } + + if (!found_cpu) { + pr_notice("gator: Adding cpu counters (based on cpuid) for %s\n", gator_cpu->core_name); + gator_events_perf_pmu_cpu_init(gator_cpu, PERF_TYPE_RAW); + } + + gator_cpus_per_core[cpu] = gator_cpu; + } + } + + /* Log the PMUs used per core */ + for (cpu = 0; cpu < ARRAY_SIZE(gator_cpuids); ++cpu) { + if (gator_cpus_per_core[cpu] != NULL) { + pr_notice("gator: Using %s (0x%lx) for cpu %i\n", gator_cpus_per_core[cpu]->core_name, gator_cpus_per_core[cpu]->cpuid, cpu); + } + } + + /* Initialize gator_attrs for non-dynamic PMUs here */ + if (attr_count > CNTMAX) { + pr_err("gator: Too many perf counters, please increase CNTMAX\n"); + return -1; + } + + if (uc_attr_count > UCCNT) { + pr_err("gator: Too many perf uncore counters, please increase UCCNT\n"); + return -1; + } + + return 0; +} + +int gator_events_perf_pmu_init(void) +{ + return gator_events_install(&gator_events_perf_pmu_interface); +} + +#else + +static int gator_events_perf_pmu_reread(void) +{ + return 0; +} + +static int gator_events_perf_pmu_create_files(struct super_block *sb, struct dentry *root) +{ + return 0; +} + +#endif diff --git a/drivers/gator/gator_events_sched.c b/drivers/gator/gator_events_sched.c new file mode 100644 index 000000000000..3e4ca5f248ea --- /dev/null +++ b/drivers/gator/gator_events_sched.c @@ -0,0 +1,135 @@ +/** + * Copyright (C) Arm Limited 2010-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#include "gator.h" +#include + +#define SCHED_SWITCH 0 +#define SCHED_TOTAL (SCHED_SWITCH+1) + +static ulong sched_switch_enabled[GATOR_CLUSTER_COUNT]; +static bool sched_switch_enabled_any; +static ulong sched_switch_key[GATOR_CLUSTER_COUNT]; +static DEFINE_PER_CPU(int[SCHED_TOTAL], schedCnt); +static DEFINE_PER_CPU(int[SCHED_TOTAL * 2], schedGet); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) +GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct task_struct *prev, struct task_struct *next)) +#else +GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(bool preempt, struct task_struct *prev, struct task_struct *next)) +#endif +{ + unsigned long flags; + + /* disable interrupts to synchronize with gator_events_sched_read() + * spinlocks not needed since percpu buffers are used + */ + local_irq_save(flags); + per_cpu(schedCnt, get_physical_cpu())[SCHED_SWITCH]++; + local_irq_restore(flags); +} + +static int gator_events_sched_create_files(struct super_block *sb, struct dentry *root) +{ + struct dentry *dir; + int i; + char buf[40]; + + /* switch */ + for (i = 0; i < gator_cluster_count; i++) { + snprintf(buf, sizeof(buf), "%s_switch", gator_clusters[i]->pmnc_name); + dir = gatorfs_mkdir(sb, root, buf); + if (!dir) + return -1; + gatorfs_create_ulong(sb, dir, "enabled", &sched_switch_enabled[i]); + gatorfs_create_ro_ulong(sb, dir, "key", &sched_switch_key[i]); + } + + return 0; +} + +static int gator_events_sched_start(void) +{ + int i; + + sched_switch_enabled_any = false; + for (i = 0; i < gator_cluster_count; i++) { + if (sched_switch_enabled[i]) { + sched_switch_enabled_any = true; + break; + } + } + + /* register tracepoints */ + if (sched_switch_enabled_any) + if (GATOR_REGISTER_TRACE(sched_switch)) + goto sched_switch_exit; + pr_debug("gator: registered scheduler event tracepoints\n"); + + return 0; + + /* unregister tracepoints on error */ +sched_switch_exit: + pr_err("gator: scheduler event tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n"); + + return -1; +} + +static void gator_events_sched_stop(void) +{ + if (sched_switch_enabled_any) + GATOR_UNREGISTER_TRACE(sched_switch); + pr_debug("gator: unregistered scheduler event tracepoints\n"); + + sched_switch_enabled_any = false; + memset(sched_switch_enabled, 0, sizeof(sched_switch_enabled)); +} + +static int gator_events_sched_read(int **buffer, bool sched_switch) +{ + unsigned long flags; + int len, value; + int cpu = get_physical_cpu(); + int cluster = gator_clusterids[cpu]; + + len = 0; + if (sched_switch_enabled[cluster]) { + local_irq_save(flags); + value = per_cpu(schedCnt, cpu)[SCHED_SWITCH]; + per_cpu(schedCnt, cpu)[SCHED_SWITCH] = 0; + local_irq_restore(flags); + per_cpu(schedGet, cpu)[len++] = sched_switch_key[cluster]; + per_cpu(schedGet, cpu)[len++] = value; + } + + if (buffer) + *buffer = per_cpu(schedGet, cpu); + + return len; +} + +static struct gator_interface gator_events_sched_interface = { + .name = "sched", + .create_files = gator_events_sched_create_files, + .start = gator_events_sched_start, + .stop = gator_events_sched_stop, + .read = gator_events_sched_read, +}; + +int gator_events_sched_init(void) +{ + int i; + + for (i = 0; i < gator_cluster_count; i++) { + sched_switch_enabled[i] = 0; + sched_switch_key[i] = gator_events_get_key(); + } + + return gator_events_install(&gator_events_sched_interface); +} diff --git a/drivers/gator/gator_fs.c b/drivers/gator/gator_fs.c new file mode 100644 index 000000000000..df37bba02d08 --- /dev/null +++ b/drivers/gator/gator_fs.c @@ -0,0 +1,375 @@ +/** + * @file gatorfs.c + * + * @remark Copyright 2002 OProfile authors + * @remark Read the file COPYING + * + * @author John Levon + * + * A simple filesystem for configuration and + * access of oprofile. + */ + +#include +#include +#include +#include +#include + + +/* Kernel version 4.6.0 removes page_cache_{get,release} and related defines. Add them back here. + (See https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/commit?id=1fa64f198b9f8d6ec0f7aec7c18dc94684391140) */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0) +# define PAGE_CACHE_SHIFT PAGE_SHIFT +# define PAGE_CACHE_SIZE PAGE_SIZE +# define PAGE_CACHE_MASK PAGE_MASK +# define PAGE_CACHE_ALIGN(addr) (((addr)+PAGE_CACHE_SIZE-1)&PAGE_CACHE_MASK) +# define page_cache_get(page) get_page(page) +# define page_cache_release(page) put_page(page) +#endif + + +#define gatorfs_MAGIC 0x24051020 +#define TMPBUFSIZE 50 +static DEFINE_SPINLOCK(gatorfs_lock); + +static struct inode *gatorfs_get_inode(struct super_block *sb, int mode) +{ + struct inode *inode = new_inode(sb); + + if (inode) { + inode->i_ino = get_next_ino(); + inode->i_mode = mode; +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0) + inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME; +#else + inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode); +#endif + } + return inode; +} + +static const struct super_operations s_ops = { + .statfs = simple_statfs, + .drop_inode = generic_delete_inode, +}; + +static ssize_t gatorfs_ulong_to_user(unsigned long val, char __user *buf, size_t count, loff_t *offset) +{ + char tmpbuf[TMPBUFSIZE]; + size_t maxlen = snprintf(tmpbuf, TMPBUFSIZE, "%lu\n", val); + + if (maxlen > TMPBUFSIZE) + maxlen = TMPBUFSIZE; + return simple_read_from_buffer(buf, count, offset, tmpbuf, maxlen); +} + +static ssize_t gatorfs_u64_to_user(u64 val, char __user *buf, size_t count, loff_t *offset) +{ + char tmpbuf[TMPBUFSIZE]; + size_t maxlen = snprintf(tmpbuf, TMPBUFSIZE, "%llu\n", val); + + if (maxlen > TMPBUFSIZE) + maxlen = TMPBUFSIZE; + return simple_read_from_buffer(buf, count, offset, tmpbuf, maxlen); +} + +static int gatorfs_ulong_from_user(unsigned long *val, char const __user *buf, size_t count) +{ + char tmpbuf[TMPBUFSIZE]; + unsigned long flags; + + if (!count) + return 0; + + if (count > TMPBUFSIZE - 1) + return -EINVAL; + + memset(tmpbuf, 0x0, TMPBUFSIZE); + + if (copy_from_user(tmpbuf, buf, count)) + return -EFAULT; + + spin_lock_irqsave(&gatorfs_lock, flags); + *val = simple_strtoul(tmpbuf, NULL, 0); + spin_unlock_irqrestore(&gatorfs_lock, flags); + return 0; +} + +static int gatorfs_u64_from_user(u64 *val, char const __user *buf, size_t count) +{ + char tmpbuf[TMPBUFSIZE]; + unsigned long flags; + + if (!count) + return 0; + + if (count > TMPBUFSIZE - 1) + return -EINVAL; + + memset(tmpbuf, 0x0, TMPBUFSIZE); + + if (copy_from_user(tmpbuf, buf, count)) + return -EFAULT; + + spin_lock_irqsave(&gatorfs_lock, flags); + *val = simple_strtoull(tmpbuf, NULL, 0); + spin_unlock_irqrestore(&gatorfs_lock, flags); + return 0; +} + +static ssize_t ulong_read_file(struct file *file, char __user *buf, size_t count, loff_t *offset) +{ + unsigned long *val = file->private_data; + + return gatorfs_ulong_to_user(*val, buf, count, offset); +} + +static ssize_t u64_read_file(struct file *file, char __user *buf, size_t count, loff_t *offset) +{ + u64 *val = file->private_data; + + return gatorfs_u64_to_user(*val, buf, count, offset); +} + +static ssize_t ulong_write_file(struct file *file, char const __user *buf, size_t count, loff_t *offset) +{ + unsigned long *value = file->private_data; + int retval; + + if (*offset) + return -EINVAL; + + retval = gatorfs_ulong_from_user(value, buf, count); + + if (retval) + return retval; + return count; +} + +static ssize_t u64_write_file(struct file *file, char const __user *buf, size_t count, loff_t *offset) +{ + u64 *value = file->private_data; + int retval; + + if (*offset) + return -EINVAL; + + retval = gatorfs_u64_from_user(value, buf, count); + + if (retval) + return retval; + return count; +} + +static int default_open(struct inode *inode, struct file *filp) +{ + if (inode->i_private) + filp->private_data = inode->i_private; + return 0; +} + +static const struct file_operations ulong_fops = { + .read = ulong_read_file, + .write = ulong_write_file, + .open = default_open, +}; + +static const struct file_operations u64_fops = { + .read = u64_read_file, + .write = u64_write_file, + .open = default_open, +}; + +static const struct file_operations ulong_ro_fops = { + .read = ulong_read_file, + .open = default_open, +}; + +static const struct file_operations u64_ro_fops = { + .read = u64_read_file, + .open = default_open, +}; + +static struct dentry *__gatorfs_create_file(struct super_block *sb, + struct dentry *root, + char const *name, + const struct file_operations *fops, + int perm) +{ + struct dentry *dentry; + struct inode *inode; + + dentry = d_alloc_name(root, name); + if (!dentry) + return NULL; + inode = gatorfs_get_inode(sb, S_IFREG | perm); + if (!inode) { + dput(dentry); + return NULL; + } + inode->i_fop = fops; + d_add(dentry, inode); + return dentry; +} + +int gatorfs_create_ulong(struct super_block *sb, struct dentry *root, + char const *name, unsigned long *val) +{ + struct dentry *d = __gatorfs_create_file(sb, root, name, + &ulong_fops, 0644); + if (!d) + return -EFAULT; + + d->d_inode->i_private = val; + return 0; +} + +static int gatorfs_create_u64(struct super_block *sb, struct dentry *root, + char const *name, u64 *val) +{ + struct dentry *d = __gatorfs_create_file(sb, root, name, + &u64_fops, 0644); + if (!d) + return -EFAULT; + + d->d_inode->i_private = val; + return 0; +} + +int gatorfs_create_ro_ulong(struct super_block *sb, struct dentry *root, + char const *name, unsigned long *val) +{ + struct dentry *d = __gatorfs_create_file(sb, root, name, + &ulong_ro_fops, 0444); + if (!d) + return -EFAULT; + + d->d_inode->i_private = val; + return 0; +} + +static int gatorfs_create_ro_u64(struct super_block *sb, struct dentry *root, + char const *name, u64 *val) +{ + struct dentry *d = + __gatorfs_create_file(sb, root, name, &u64_ro_fops, 0444); + if (!d) + return -EFAULT; + + d->d_inode->i_private = val; + return 0; +} + +static ssize_t atomic_read_file(struct file *file, char __user *buf, size_t count, loff_t *offset) +{ + atomic_t *val = file->private_data; + + return gatorfs_ulong_to_user(atomic_read(val), buf, count, offset); +} + +static const struct file_operations atomic_ro_fops = { + .read = atomic_read_file, + .open = default_open, +}; + +static int gatorfs_create_file_data(struct super_block *sb, struct dentry *root, + char const *name, const struct file_operations *fops, + void * private_data) +{ + struct dentry * dentry = __gatorfs_create_file(sb, root, name, fops, 0644); + if (!dentry) + return -EFAULT; + + dentry->d_inode->i_private = private_data; + return 0; +} + +static int gatorfs_create_file(struct super_block *sb, struct dentry *root, + char const *name, const struct file_operations *fops) +{ + return gatorfs_create_file_data(sb, root, name, fops, NULL); +} + +static int gatorfs_create_file_perm(struct super_block *sb, struct dentry *root, + char const *name, + const struct file_operations *fops, int perm) +{ + if (!__gatorfs_create_file(sb, root, name, fops, perm)) + return -EFAULT; + return 0; +} + +struct dentry *gatorfs_mkdir(struct super_block *sb, + struct dentry *root, char const *name) +{ + struct dentry *dentry; + struct inode *inode; + + dentry = d_alloc_name(root, name); + if (!dentry) + return NULL; + inode = gatorfs_get_inode(sb, S_IFDIR | 0755); + if (!inode) { + dput(dentry); + return NULL; + } + inode->i_op = &simple_dir_inode_operations; + inode->i_fop = &simple_dir_operations; + d_add(dentry, inode); + return dentry; +} + +static int gatorfs_fill_super(struct super_block *sb, void *data, int silent) +{ + struct inode *root_inode; + struct dentry *root_dentry; + + sb->s_blocksize = PAGE_CACHE_SIZE; + sb->s_blocksize_bits = PAGE_CACHE_SHIFT; + sb->s_magic = gatorfs_MAGIC; + sb->s_op = &s_ops; + sb->s_time_gran = 1; + + root_inode = gatorfs_get_inode(sb, S_IFDIR | 0755); + if (!root_inode) + return -ENOMEM; + root_inode->i_op = &simple_dir_inode_operations; + root_inode->i_fop = &simple_dir_operations; + + root_dentry = d_make_root(root_inode); + + if (!root_dentry) { + return -ENOMEM; + } + + sb->s_root = root_dentry; + + gator_op_create_files(sb, root_dentry); + + return 0; +} + +static struct dentry *gatorfs_mount(struct file_system_type *fs_type, + int flags, const char *dev_name, void *data) +{ + return mount_nodev(fs_type, flags, data, gatorfs_fill_super); +} + +static struct file_system_type gatorfs_type = { + .owner = THIS_MODULE, + .name = "gatorfs", + .mount = gatorfs_mount, + + .kill_sb = kill_litter_super, +}; + +static int __init gatorfs_register(void) +{ + return register_filesystem(&gatorfs_type); +} + +static void gatorfs_unregister(void) +{ + unregister_filesystem(&gatorfs_type); +} diff --git a/drivers/gator/gator_hrtimer_gator.c b/drivers/gator/gator_hrtimer_gator.c new file mode 100644 index 000000000000..2356e4cee060 --- /dev/null +++ b/drivers/gator/gator_hrtimer_gator.c @@ -0,0 +1,80 @@ +/** + * Copyright (C) Arm Limited 2011-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +static void (*callback)(void); +static DEFINE_PER_CPU(struct hrtimer, percpu_hrtimer); +static DEFINE_PER_CPU(ktime_t, hrtimer_expire); +static DEFINE_PER_CPU(int, hrtimer_is_active); +static ktime_t profiling_interval; +static void gator_hrtimer_online(void); +static void gator_hrtimer_offline(void); + +static enum hrtimer_restart gator_hrtimer_notify(struct hrtimer *hrtimer) +{ + int cpu = get_logical_cpu(); + + hrtimer_forward(hrtimer, per_cpu(hrtimer_expire, cpu), profiling_interval); + per_cpu(hrtimer_expire, cpu) = ktime_add(per_cpu(hrtimer_expire, cpu), profiling_interval); + (*callback)(); + return HRTIMER_RESTART; +} + +static void gator_hrtimer_online(void) +{ + int cpu = get_logical_cpu(); + struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu); + + if (per_cpu(hrtimer_is_active, cpu) || (ktime_to_ns(profiling_interval) == 0)) + return; + + per_cpu(hrtimer_is_active, cpu) = 1; + hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); + hrtimer->function = gator_hrtimer_notify; +#ifdef CONFIG_PREEMPT_RT_BASE + hrtimer->irqsafe = 1; +#endif + per_cpu(hrtimer_expire, cpu) = ktime_add(hrtimer->base->get_time(), profiling_interval); + hrtimer_start(hrtimer, per_cpu(hrtimer_expire, cpu), HRTIMER_MODE_ABS_PINNED); +} + +static void gator_hrtimer_offline(void) +{ + int cpu = get_logical_cpu(); + struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu); + + if (!per_cpu(hrtimer_is_active, cpu)) + return; + + per_cpu(hrtimer_is_active, cpu) = 0; + hrtimer_cancel(hrtimer); +} + +static int gator_hrtimer_init(int interval, void (*func)(void)) +{ + int cpu; + + (callback) = (func); + + for_each_present_cpu(cpu) { + per_cpu(hrtimer_is_active, cpu) = 0; + } + + /* calculate profiling interval */ + if (interval > 0) + profiling_interval = ns_to_ktime(1000000000UL / interval); + else + profiling_interval = ns_to_ktime(0); + + return 0; +} + +static void gator_hrtimer_shutdown(void) +{ + /* empty */ +} diff --git a/drivers/gator/gator_iks.c b/drivers/gator/gator_iks.c new file mode 100644 index 000000000000..0abc63d8b96a --- /dev/null +++ b/drivers/gator/gator_iks.c @@ -0,0 +1,195 @@ +/** + * Copyright (C) Arm Limited 2013-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#if GATOR_IKS_SUPPORT + +#include +#include +#include +#include + +static bool map_cpuids; +static int mpidr_cpuids[NR_CPUS]; +static const struct gator_cpu *mpidr_cpus[NR_CPUS]; +static int __lcpu_to_pcpu[NR_CPUS]; + +static const struct gator_cpu *gator_find_cpu_by_dt_name(const char *const name) +{ + const struct gator_cpu *gator_cpu; + + list_for_each_entry(gator_cpu, &gator_cpus, list) { + if (gator_cpu->dt_name != NULL && strcmp(gator_cpu->dt_name, name) == 0) + return gator_cpu; + } + + return NULL; +} + +static void calc_first_cluster_size(void) +{ + int len; + const u32 *val; + const char *compatible; + struct device_node *cn = NULL; + int mpidr_cpuids_count = 0; + + /* Zero is a valid cpuid, so initialize the array to 0xff's */ + memset(&mpidr_cpuids, 0xff, sizeof(mpidr_cpuids)); + memset(&mpidr_cpus, 0, sizeof(mpidr_cpus)); + + while ((cn = of_find_node_by_type(cn, "cpu"))) { + BUG_ON(mpidr_cpuids_count >= NR_CPUS); + + val = of_get_property(cn, "reg", &len); + if (!val || len != 4) { + pr_err("%s missing reg property\n", cn->full_name); + continue; + } + compatible = of_get_property(cn, "compatible", NULL); + if (compatible == NULL) { + pr_err("%s missing compatible property\n", cn->full_name); + continue; + } + + mpidr_cpuids[mpidr_cpuids_count] = be32_to_cpup(val); + mpidr_cpus[mpidr_cpuids_count] = gator_find_cpu_by_dt_name(compatible); + ++mpidr_cpuids_count; + } + + map_cpuids = (mpidr_cpuids_count == nr_cpu_ids); +} + +static int linearize_mpidr(int mpidr) +{ + int i; + + for (i = 0; i < nr_cpu_ids; ++i) { + if (mpidr_cpuids[i] == mpidr) + return i; + } + + BUG(); +} + +int lcpu_to_pcpu(const int lcpu) +{ + int pcpu; + + if (!map_cpuids) + return lcpu; + + BUG_ON(lcpu >= nr_cpu_ids || lcpu < 0); + pcpu = __lcpu_to_pcpu[lcpu]; + BUG_ON(pcpu >= nr_cpu_ids || pcpu < 0); + return pcpu; +} + +int pcpu_to_lcpu(const int pcpu) +{ + int lcpu; + + if (!map_cpuids) + return pcpu; + + BUG_ON(pcpu >= nr_cpu_ids || pcpu < 0); + for (lcpu = 0; lcpu < nr_cpu_ids; ++lcpu) { + if (__lcpu_to_pcpu[lcpu] == pcpu) { + BUG_ON(lcpu >= nr_cpu_ids || lcpu < 0); + return lcpu; + } + } + BUG(); +} + +static void gator_update_cpu_mapping(u32 cpu_hwid) +{ + int lcpu = smp_processor_id(); + int pcpu = linearize_mpidr(cpu_hwid & MPIDR_HWID_BITMASK); + + BUG_ON(lcpu >= nr_cpu_ids || lcpu < 0); + BUG_ON(pcpu >= nr_cpu_ids || pcpu < 0); + __lcpu_to_pcpu[lcpu] = pcpu; +} + +GATOR_DEFINE_PROBE(cpu_migrate_begin, TP_PROTO(u64 timestamp, u32 cpu_hwid)) +{ + const int cpu = get_physical_cpu(); + + gator_timer_offline((void *)1); + gator_timer_offline_dispatch(cpu, true); +} + +GATOR_DEFINE_PROBE(cpu_migrate_finish, TP_PROTO(u64 timestamp, u32 cpu_hwid)) +{ + int cpu; + + gator_update_cpu_mapping(cpu_hwid); + + /* get_physical_cpu must be called after gator_update_cpu_mapping */ + cpu = get_physical_cpu(); + gator_timer_online_dispatch(cpu, true); + gator_timer_online((void *)1); +} + +GATOR_DEFINE_PROBE(cpu_migrate_current, TP_PROTO(u64 timestamp, u32 cpu_hwid)) +{ + gator_update_cpu_mapping(cpu_hwid); +} + +static void gator_send_iks_core_names(void) +{ + int cpu; + /* Send the cpu names */ + preempt_disable(); + for (cpu = 0; cpu < nr_cpu_ids; ++cpu) { + if (mpidr_cpus[cpu] != NULL) + gator_send_core_name(cpu, mpidr_cpus[cpu]->cpuid); + } + preempt_enable(); +} + +static int gator_migrate_start(void) +{ + int retval = 0; + + if (!map_cpuids) + return retval; + + if (retval == 0) + retval = GATOR_REGISTER_TRACE(cpu_migrate_begin); + if (retval == 0) + retval = GATOR_REGISTER_TRACE(cpu_migrate_finish); + if (retval == 0) + retval = GATOR_REGISTER_TRACE(cpu_migrate_current); + if (retval == 0) { + /* Initialize the logical to physical cpu mapping */ + memset(&__lcpu_to_pcpu, 0xff, sizeof(__lcpu_to_pcpu)); + bL_switcher_trace_trigger(); + } + return retval; +} + +static void gator_migrate_stop(void) +{ + if (!map_cpuids) + return; + + GATOR_UNREGISTER_TRACE(cpu_migrate_current); + GATOR_UNREGISTER_TRACE(cpu_migrate_finish); + GATOR_UNREGISTER_TRACE(cpu_migrate_begin); +} + +#else + +#define calc_first_cluster_size() +#define gator_send_iks_core_names() +#define gator_migrate_start() 0 +#define gator_migrate_stop() + +#endif diff --git a/drivers/gator/gator_main.c b/drivers/gator/gator_main.c new file mode 100644 index 000000000000..abe52ab57158 --- /dev/null +++ b/drivers/gator/gator_main.c @@ -0,0 +1,1501 @@ +/** + * Copyright (C) Arm Limited 2010-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +/* This version must match the gator daemon version */ +#define PROTOCOL_VERSION 670 +static unsigned long gator_protocol_version = PROTOCOL_VERSION; + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) +#include +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 15, 0) +#include +#endif + +#include "gator.h" +#include "generated_gator_src_md5.h" + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0) +#error Kernels prior to 3.4 not supported. DS-5 v5.21 and earlier supported 2.6.32 and later. +#endif + +#if defined(MODULE) && !defined(CONFIG_MODULES) +#error Cannot build a module against a kernel that does not support modules. To resolve, either rebuild the kernel to support modules or build gator as part of the kernel. +#endif + +#if !defined(CONFIG_GENERIC_TRACER) && !defined(CONFIG_TRACING) +#error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined +#endif + +#ifndef CONFIG_PROFILING +#error gator requires the kernel to have CONFIG_PROFILING defined +#endif + +#ifndef CONFIG_HIGH_RES_TIMERS +#error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined to support PC sampling +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) && defined(__arm__) && defined(CONFIG_SMP) && !defined(CONFIG_LOCAL_TIMERS) +#error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems +#endif + +#if !(GATOR_PERF_PMU_SUPPORT) +#ifndef CONFIG_PERF_EVENTS +#error gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters +#elif !defined CONFIG_HW_PERF_EVENTS +#error gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters +#endif +#endif + +/****************************************************************************** + * DEFINES + ******************************************************************************/ +#define SUMMARY_BUFFER_SIZE (1*1024) +#define BACKTRACE_BUFFER_SIZE (128*1024) +#define NAME_BUFFER_SIZE (64*1024) +#define COUNTER_BUFFER_SIZE (64*1024) /* counters have the core as part of the data and the core value in the frame header may be discarded */ +#define BLOCK_COUNTER_BUFFER_SIZE (128*1024) +#define ANNOTATE_BUFFER_SIZE (128*1024) /* annotate counters have the core as part of the data and the core value in the frame header may be discarded */ +#define SCHED_TRACE_BUFFER_SIZE (128*1024) +#define IDLE_BUFFER_SIZE (32*1024) /* idle counters have the core as part of the data and the core value in the frame header may be discarded */ +#define ACTIVITY_BUFFER_SIZE (128*1024) + +#define NO_COOKIE 0U +#define UNRESOLVED_COOKIE ~0U + +#define FRAME_SUMMARY 1 +#define FRAME_BACKTRACE 2 +#define FRAME_NAME 3 +#define FRAME_COUNTER 4 +#define FRAME_BLOCK_COUNTER 5 +#define FRAME_ANNOTATE 6 +#define FRAME_SCHED_TRACE 7 +#define FRAME_IDLE 9 +#define FRAME_ACTIVITY 13 + +#define MESSAGE_END_BACKTRACE 1 + +/* Name Frame Messages */ +#define MESSAGE_COOKIE 1 +#define MESSAGE_THREAD_NAME 2 + +/* Scheduler Trace Frame Messages */ +#define MESSAGE_SCHED_SWITCH 1 +#define MESSAGE_SCHED_EXIT 2 + +/* Summary Frame Messages */ +#define MESSAGE_SUMMARY 1 +#define MESSAGE_CORE_NAME 3 + +/* Activity Frame Messages */ +#define MESSAGE_LINK 1 +#define MESSAGE_SWITCH 2 +#define MESSAGE_EXIT 3 + +#define MAXSIZE_PACK32 5 +#define MAXSIZE_PACK64 10 + +#define FRAME_HEADER_SIZE 3 + +#if defined(__arm__) +#define PC_REG regs->ARM_pc +#elif defined(__aarch64__) +#define PC_REG regs->pc +#else +#define PC_REG regs->ip +#endif + +enum { + SUMMARY_BUF, + BACKTRACE_BUF, + NAME_BUF, + COUNTER_BUF, + BLOCK_COUNTER_BUF, + ANNOTATE_BUF, + SCHED_TRACE_BUF, + IDLE_BUF, + ACTIVITY_BUF, + NUM_GATOR_BUFS +}; + +/****************************************************************************** + * Globals + ******************************************************************************/ +static unsigned long gator_cpu_cores; +/* Size of the largest buffer. Effectively constant, set in gator_op_create_files */ +static unsigned long userspace_buffer_size; +static unsigned long gator_backtrace_depth; +/* How often to commit the buffers for live in nanoseconds */ +static u64 gator_live_rate; + +static unsigned long gator_started; +static u64 gator_monotonic_started; +static u64 gator_sync_time; +static u64 gator_hibernate_time; +static unsigned long gator_buffer_opened; +static unsigned long gator_timer_count; +static unsigned long gator_response_type; +static DEFINE_MUTEX(start_mutex); +static DEFINE_MUTEX(gator_buffer_mutex); + +static bool event_based_sampling; + +static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait); +static DECLARE_WAIT_QUEUE_HEAD(gator_annotate_wait); +static struct timer_list gator_buffer_wake_up_timer; +static bool gator_buffer_wake_run; +/* Initialize semaphore unlocked to initialize memory values */ +static DEFINE_SEMAPHORE(gator_buffer_wake_sem); +static struct task_struct *gator_buffer_wake_thread; +static LIST_HEAD(gator_events); + +static DEFINE_PER_CPU(u64, last_timestamp); + +static bool printed_monotonic_warning; + +static u32 gator_cpuids[NR_CPUS]; +int gator_clusterids[NR_CPUS]; +static bool sent_core_name[NR_CPUS]; + +static DEFINE_PER_CPU(bool, in_scheduler_context); + +/****************************************************************************** + * Prototypes + ******************************************************************************/ +static void gator_emit_perf_time(u64 time); +static void gator_op_create_files(struct super_block *sb, struct dentry *root); +static void gator_backtrace_handler(struct pt_regs *const regs); +static int gator_events_perf_pmu_reread(void); +static int gator_events_perf_pmu_create_files(struct super_block *sb, struct dentry *root); +static void gator_trace_power_init(void); +static int gator_trace_power_create_files(struct super_block *sb, struct dentry *root); +static int sched_trace_create_files(struct super_block *sb, struct dentry *root); +static void gator_trace_sched_init(void); + +/* gator_buffer is protected by being per_cpu and by having IRQs + * disabled when writing to it. Most marshal_* calls take care of this + * except for marshal_cookie*, marshal_backtrace* and marshal_frame + * where the caller is responsible for doing so. No synchronization is + * needed with the backtrace buffer as it is per cpu and is only used + * from the hrtimer. The annotate_lock must be held when using the + * annotation buffer as it is not per cpu. collect_counters which is + * the sole writer to the block counter frame is additionally + * protected by the per cpu collecting flag. + */ + +/* Size of the buffer, must be a power of 2. Effectively constant, set in gator_op_setup. */ +static uint32_t gator_buffer_size[NUM_GATOR_BUFS]; +/* gator_buffer_size - 1, bitwise and with pos to get offset into the array. Effectively constant, set in gator_op_setup. */ +static uint32_t gator_buffer_mask[NUM_GATOR_BUFS]; +/* Read position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are read by userspace in userspace_buffer_read */ +static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read); +/* Write position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are written to the buffer */ +static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write); +/* Commit position in the buffer. Initialized to zero in gator_op_setup and incremented after a frame is ready to be read by userspace */ +static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_commit); +/* If set to false, decreases the number of bytes returned by + * buffer_bytes_available. Set in buffer_check_space if no space is + * remaining. Initialized to true in gator_op_setup. This means that + * if we run out of space, continue to report that no space is + * available until bytes are read by userspace + */ +static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available); +/* The buffer. Allocated in gator_op_setup */ +static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer); +/* The time after which the buffer should be committed for live display */ +static DEFINE_PER_CPU(u64, gator_buffer_commit_time); + +/* List of all gator events - new events must be added to this list */ +#define GATOR_EVENTS_LIST \ + GATOR_EVENT(gator_events_block_init) \ + GATOR_EVENT(gator_events_ccn504_init) \ + GATOR_EVENT(gator_events_irq_init) \ + GATOR_EVENT(gator_events_l2c310_init) \ + GATOR_EVENT(gator_events_mali_init) \ + GATOR_EVENT(gator_events_mali_midgard_hw_init) \ + GATOR_EVENT(gator_events_mali_midgard_init) \ + GATOR_EVENT(gator_events_meminfo_init) \ + GATOR_EVENT(gator_events_mmapped_init) \ + GATOR_EVENT(gator_events_net_init) \ + GATOR_EVENT(gator_events_perf_pmu_init) \ + GATOR_EVENT(gator_events_sched_init) \ + +#define GATOR_EVENT(EVENT_INIT) __weak int EVENT_INIT(void); +GATOR_EVENTS_LIST +#undef GATOR_EVENT + +static int (*gator_events_list[])(void) = { +#define GATOR_EVENT(EVENT_INIT) EVENT_INIT, +GATOR_EVENTS_LIST +#undef GATOR_EVENT +}; + +/****************************************************************************** + * Application Includes + ******************************************************************************/ +#include "gator_fs.c" +#include "gator_pmu.c" +#include "gator_buffer_write.c" +#include "gator_buffer.c" +#include "gator_marshaling.c" +#include "gator_hrtimer_gator.c" +#include "gator_cookies.c" +#include "gator_annotate.c" +#include "gator_trace_sched.c" +#include "gator_trace_power.c" +#include "gator_trace_gpu.c" +#include "gator_backtrace.c" +#include "gator_events_perf_pmu.c" + +/****************************************************************************** + * Misc + ******************************************************************************/ + +MODULE_PARM_DESC(gator_src_md5, "Gator driver source code md5sum"); +module_param_named(src_md5, gator_src_md5, charp, 0444); + +u32 gator_cpuid(void) +{ +#if defined(__arm__) || defined(__aarch64__) + u32 val; +#if !defined(__aarch64__) + asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val)); +#else + asm volatile("mrs %0, midr_el1" : "=r" (val)); +#endif + return ((val & 0xff000000) >> 12) | ((val & 0xfff0) >> 4); +#else + return OTHER; +#endif +} + +static DECLARE_TIMER_HANDLER(gator_buffer_wake_up) +{ + wake_up(&gator_buffer_wait); +} + +static int gator_buffer_wake_func(void *data) +{ + for (;;) { + if (down_killable(&gator_buffer_wake_sem)) + break; + + /* Eat up any pending events */ + while (!down_trylock(&gator_buffer_wake_sem)) + ; + + if (!gator_buffer_wake_run) + break; + + gator_buffer_wake_up(0); + } + + return 0; +} + +/****************************************************************************** + * Commit interface + ******************************************************************************/ +static bool buffer_commit_ready(int prev_cpu, int prev_buftype, int *out_cpu, int * out_buftype) +{ + int cpu_x, x; + + // simple sort of QOS/fair scheduling of buffer checking. we scan starting at the next item after the last successful one + // up to the end, and if nothing is found from the start upto and including the last successful one. + // that way we do not favour lower number cpu or lower number buffer. + + // do everything after (prev_cpu:prev_buftype) + for (x = 0; x < NUM_GATOR_BUFS; x++) { + for_each_present_cpu(cpu_x) { + if ((cpu_x > prev_cpu) || ((cpu_x == prev_cpu) && (x > prev_buftype))) { + if (per_cpu(gator_buffer_commit, cpu_x)[x] != per_cpu(gator_buffer_read, cpu_x)[x]) { + *out_cpu = cpu_x; + *out_buftype = x; + return true; + } + } + else { + goto low_half; + } + } + } + + + // now everything upto and including (prev_cpu:prev_buftype) +low_half: + + for (x = 0; x < NUM_GATOR_BUFS; x++) { + for_each_present_cpu(cpu_x) { + if ((cpu_x < prev_cpu) || ((cpu_x == prev_cpu) && (x <= prev_buftype))) { + if (per_cpu(gator_buffer_commit, cpu_x)[x] != per_cpu(gator_buffer_read, cpu_x)[x]) { + *out_cpu = cpu_x; + *out_buftype = x; + return true; + } + } + else { + goto not_found; + } + } + } + + // nothing found +not_found: + + *out_cpu = -1; + *out_buftype = -1; + return false; +} + +/****************************************************************************** + * hrtimer interrupt processing + ******************************************************************************/ +static void gator_timer_interrupt(void) +{ + struct pt_regs *const regs = get_irq_regs(); + + gator_backtrace_handler(regs); +} + +static void gator_backtrace_handler(struct pt_regs *const regs) +{ + u64 time = gator_get_time(); + int cpu = get_physical_cpu(); + + /* Output backtrace */ + gator_add_sample(cpu, regs, time); + + /* Collect counters */ + if (!per_cpu(collecting, cpu)) + collect_counters(time, current, false); + + /* No buffer flushing occurs during sched switch for RT-Preempt full. The block counter frame will be flushed by collect_counters, but the sched buffer needs to be explicitly flushed */ +#ifdef CONFIG_PREEMPT_RT_FULL + buffer_check(cpu, SCHED_TRACE_BUF, time); +#endif +} + +static int gator_running; + +/* This function runs in interrupt context and on the appropriate core */ +static void gator_timer_offline(void *migrate) +{ + struct gator_interface *gi; + int i, len, cpu = get_physical_cpu(); + int *buffer; + u64 time; + + gator_trace_sched_offline(); + gator_trace_power_offline(); + + if (!migrate) + gator_hrtimer_offline(); + + /* Offline any events and output counters */ + time = gator_get_time(); + if (marshal_event_header(time)) { + list_for_each_entry(gi, &gator_events, list) { + if (gi->offline) { + len = gi->offline(&buffer, migrate); + if (len < 0) + pr_err("gator: offline failed for %s\n", gi->name); + marshal_event(len, buffer); + } + } + /* Only check after writing all counters so that time and corresponding counters appear in the same frame */ + buffer_check(cpu, BLOCK_COUNTER_BUF, time); + } + + /* Flush all buffers on this core */ + for (i = 0; i < NUM_GATOR_BUFS; i++) + gator_commit_buffer(cpu, i, time); +} + +/* This function runs in interrupt context and may be running on a core other than core 'cpu' */ +static void gator_timer_offline_dispatch(int cpu, bool migrate) +{ + struct gator_interface *gi; + + list_for_each_entry(gi, &gator_events, list) { + if (gi->offline_dispatch) + gi->offline_dispatch(cpu, migrate); + } +} + +static void gator_timer_stop(void) +{ + int cpu; + + if (gator_running) { + on_each_cpu(gator_timer_offline, NULL, 1); + for_each_online_cpu(cpu) { + gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false); + } + + gator_running = 0; + gator_hrtimer_shutdown(); + } +} + +static int gator_get_clusterid(const u32 cpuid) +{ + int i; + + for (i = 0; i < gator_cluster_count; i++) { + if (gator_clusters[i]->cpuid == cpuid) + return i; + } + + return 0; +} + +static void gator_send_core_name(const int cpu, const u32 cpuid) +{ +#if defined(__arm__) || defined(__aarch64__) + if (!sent_core_name[cpu] || (cpuid != gator_cpuids[cpu])) { + const struct gator_cpu *const gator_cpu = gator_find_cpu_by_cpuid(cpuid); + const char *core_name = NULL; + char core_name_buf[32]; + + /* Save off this cpuid */ + gator_cpuids[cpu] = cpuid; + gator_clusterids[cpu] = gator_get_clusterid(cpuid); + if (gator_cpu != NULL) { + core_name = gator_cpu->core_name; + } else { + if (cpuid == -1) + snprintf(core_name_buf, sizeof(core_name_buf), "Unknown"); + else + snprintf(core_name_buf, sizeof(core_name_buf), "Unknown (0x%.5x)", cpuid); + core_name = core_name_buf; + } + + marshal_core_name(cpu, cpuid, core_name); + sent_core_name[cpu] = true; + } +#endif +} + +static void gator_read_cpuid(void *arg) +{ + const u32 cpuid = gator_cpuid(); + const int cpu = get_physical_cpu(); + + pr_notice("gator: Detected CPUID for %i as 0x%x\n", cpu, cpuid); + + gator_cpuids[cpu] = cpuid; + gator_clusterids[cpu] = gator_get_clusterid(cpuid); +} + +/* This function runs in interrupt context and on the appropriate core */ +static void gator_timer_online(void *migrate) +{ + struct gator_interface *gi; + int len, cpu = get_physical_cpu(); + int *buffer; + u64 time; + + /* Send what is currently running on this core */ + marshal_sched_trace_switch(current->pid, 0); + + gator_trace_power_online(); + + /* online any events and output counters */ + time = gator_get_time(); + if (marshal_event_header(time)) { + list_for_each_entry(gi, &gator_events, list) { + if (gi->online) { + len = gi->online(&buffer, migrate); + if (len < 0) + pr_err("gator: online failed for %s\n", gi->name); + marshal_event(len, buffer); + } + } + /* Only check after writing all counters so that time and corresponding counters appear in the same frame */ + buffer_check(cpu, BLOCK_COUNTER_BUF, time); + } + + if (!migrate) + gator_hrtimer_online(); + + gator_send_core_name(cpu, gator_cpuid()); +} + +/* This function runs in interrupt context and may be running on a core other than core 'cpu' */ +static void gator_timer_online_dispatch(int cpu, bool migrate) +{ + struct gator_interface *gi; + + list_for_each_entry(gi, &gator_events, list) { + if (gi->online_dispatch) + gi->online_dispatch(cpu, migrate); + } +} + +#include "gator_iks.c" + +static int gator_timer_start(unsigned long sample_rate) +{ + int cpu; + + if (gator_running) { + pr_notice("gator: already running\n"); + return 0; + } + + gator_running = 1; + + /* event based sampling trumps hr timer based sampling */ + if (event_based_sampling) + sample_rate = 0; + + if (gator_hrtimer_init(sample_rate, gator_timer_interrupt) == -1) + return -1; + + /* Send off the previously saved cpuids */ + for_each_present_cpu(cpu) { + preempt_disable(); + gator_send_core_name(cpu, gator_cpuids[cpu]); + preempt_enable(); + } + + gator_send_iks_core_names(); + for_each_online_cpu(cpu) { + gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false); + } + on_each_cpu(gator_timer_online, NULL, 1); + + return 0; +} + +u64 gator_get_time(void) +{ + struct timespec ts; + u64 timestamp; + u64 prev_timestamp; + u64 delta; + int cpu = smp_processor_id(); + + /* Match clock_gettime(CLOCK_MONOTONIC_RAW, &ts) from userspace */ + getrawmonotonic(&ts); + timestamp = timespec_to_ns(&ts); + + /* getrawmonotonic is not monotonic on all systems. Detect and + * attempt to correct these cases. up to 0.5ms delta has been seen + * on some systems, which can skew Streamline data when viewing at + * high resolution. This doesn't work well with interrupts, but that + * it's OK - the real concern is to catch big jumps in time + */ + prev_timestamp = per_cpu(last_timestamp, cpu); + if (prev_timestamp <= timestamp) { + per_cpu(last_timestamp, cpu) = timestamp; + } else { + delta = prev_timestamp - timestamp; + /* Log the error once */ + if (!printed_monotonic_warning && delta > 500000) { + pr_err("%s: getrawmonotonic is not monotonic cpu: %i delta: %lli\nSkew in Streamline data may be present at the fine zoom levels\n", __func__, cpu, delta); + printed_monotonic_warning = true; + } + timestamp = prev_timestamp; + } + + return timestamp - gator_monotonic_started; +} + +static void gator_emit_perf_time(u64 time) +{ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 10, 0) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 2, 0) + if (time >= gator_sync_time) { + marshal_event_single64(0, -1, local_clock()); + gator_sync_time += NSEC_PER_SEC; + if (gator_live_rate <= 0) + gator_commit_buffer(get_physical_cpu(), COUNTER_BUF, time); + } +#endif +} + +/****************************************************************************** + * cpu hotplug and pm notifiers + ******************************************************************************/ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) + +static enum cpuhp_state gator_cpuhp_online; + +static int gator_cpuhp_notify_online(unsigned int cpu) +{ + gator_timer_online_dispatch(cpu, false); + smp_call_function_single(cpu, gator_timer_online, NULL, 1); + return 0; +} + +static int gator_cpuhp_notify_offline(unsigned int cpu) +{ + smp_call_function_single(cpu, gator_timer_offline, NULL, 1); + gator_timer_offline_dispatch(cpu, false); + return 0; +} + +static int gator_register_hotcpu_notifier(void) +{ + int retval; + + retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "gator/cpuhotplug:online", gator_cpuhp_notify_online, gator_cpuhp_notify_offline); + if (retval >= 0) { + gator_cpuhp_online = retval; + retval = 0; + } + return retval; +} + +static void gator_unregister_hotcpu_notifier(void) +{ + cpuhp_remove_state(gator_cpuhp_online); +} + +#else + +static int gator_hotcpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) +{ + int cpu = lcpu_to_pcpu((long)hcpu); + + switch (action) { + case CPU_DOWN_PREPARE: + case CPU_DOWN_PREPARE_FROZEN: + smp_call_function_single(cpu, gator_timer_offline, NULL, 1); + gator_timer_offline_dispatch(cpu, false); + break; + case CPU_ONLINE: + case CPU_ONLINE_FROZEN: + gator_timer_online_dispatch(cpu, false); + smp_call_function_single(cpu, gator_timer_online, NULL, 1); + break; + } + + return NOTIFY_OK; +} + +static struct notifier_block __refdata gator_hotcpu_notifier = { + .notifier_call = gator_hotcpu_notify, +}; + +static int gator_register_hotcpu_notifier(void) +{ + return register_hotcpu_notifier(&gator_hotcpu_notifier); +} + +static void gator_unregister_hotcpu_notifier(void) +{ + unregister_hotcpu_notifier(&gator_hotcpu_notifier); +} + +#endif + +/* n.b. calling "on_each_cpu" only runs on those that are online. + * Registered linux events are not disabled, so their counters will + * continue to collect + */ +static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy) +{ + int cpu; + struct timespec ts; + + switch (event) { + case PM_HIBERNATION_PREPARE: + case PM_SUSPEND_PREPARE: + gator_unregister_hotcpu_notifier(); + unregister_scheduler_tracepoints(); + on_each_cpu(gator_timer_offline, NULL, 1); + for_each_online_cpu(cpu) { + gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false); + } + + /* Record the wallclock hibernate time */ + getnstimeofday(&ts); + gator_hibernate_time = timespec_to_ns(&ts) - gator_get_time(); + break; + case PM_POST_HIBERNATION: + case PM_POST_SUSPEND: + /* Adjust gator_monotonic_started for the time spent sleeping, as gator_get_time does not account for it */ + if (gator_hibernate_time > 0) { + getnstimeofday(&ts); + gator_monotonic_started += gator_hibernate_time + gator_get_time() - timespec_to_ns(&ts); + gator_hibernate_time = 0; + } + + for_each_online_cpu(cpu) { + gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false); + } + on_each_cpu(gator_timer_online, NULL, 1); + register_scheduler_tracepoints(); + gator_register_hotcpu_notifier(); + break; + } + + return NOTIFY_OK; +} + +static struct notifier_block gator_pm_notifier = { + .notifier_call = gator_pm_notify, +}; + +static int gator_notifier_start(void) +{ + int retval = gator_register_hotcpu_notifier(); + if (retval == 0) + retval = register_pm_notifier(&gator_pm_notifier); + return retval; +} + +static void gator_notifier_stop(void) +{ + unregister_pm_notifier(&gator_pm_notifier); + gator_unregister_hotcpu_notifier(); +} + +/****************************************************************************** + * Main + ******************************************************************************/ +static void gator_summary(void) +{ + u64 timestamp, uptime; + struct timespec ts; + char uname_buf[100]; + + snprintf(uname_buf, sizeof(uname_buf), "%s %s %s %s %s GNU/Linux", utsname()->sysname, utsname()->nodename, utsname()->release, utsname()->version, utsname()->machine); + + getnstimeofday(&ts); + timestamp = timespec_to_ns(&ts); + + /* Similar to reading /proc/uptime from fs/proc/uptime.c, calculate uptime */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 11, 0) + { + void (*m2b)(struct timespec *ts); + + do_posix_clock_monotonic_gettime(&ts); + /* monotonic_to_bootbased is not defined for some versions of Android */ + m2b = symbol_get(monotonic_to_bootbased); + if (m2b) + m2b(&ts); + } +#else + get_monotonic_boottime(&ts); +#endif + uptime = timespec_to_ns(&ts); + + /* Disable preemption as gator_get_time calls smp_processor_id to verify time is monotonic */ + preempt_disable(); + /* Set monotonic_started to zero as gator_get_time is uptime minus monotonic_started */ + gator_monotonic_started = 0; + gator_monotonic_started = gator_get_time(); + + marshal_summary(timestamp, uptime, gator_monotonic_started, uname_buf); + gator_sync_time = 0; + gator_emit_perf_time(gator_monotonic_started); + /* Always flush COUNTER_BUF so that the initial perf_time is received before it's used */ + gator_commit_buffer(get_physical_cpu(), COUNTER_BUF, 0); + preempt_enable(); +} + +int gator_events_install(struct gator_interface *interface) +{ + list_add_tail(&interface->list, &gator_events); + + return 0; +} + +int gator_events_get_key(void) +{ + /* key 0 is reserved as a timestamp. key 1 is reserved as the marker + * for thread specific counters. key 2 is reserved as the marker for + * core. Odd keys are assigned by the driver, even keys by the + * daemon. + */ + static int key = 3; + const int ret = key; + + key += 2; + return ret; +} + +static int gator_init(void) +{ + calc_first_cluster_size(); + + return 0; +} + +static void gator_exit(void) +{ + struct gator_interface *gi; + + list_for_each_entry(gi, &gator_events, list) + if (gi->shutdown) + gi->shutdown(); +} + +static int gator_start(void) +{ + unsigned long cpu, i; + struct gator_interface *gi; + + gator_buffer_wake_run = true; + gator_buffer_wake_thread = kthread_run(gator_buffer_wake_func, NULL, "gator_bwake"); + if (IS_ERR(gator_buffer_wake_thread)) + goto bwake_failure; + + if (gator_migrate_start()) + goto migrate_failure; + + /* Initialize the buffer with the frame type and core */ + for_each_present_cpu(cpu) { + for (i = 0; i < NUM_GATOR_BUFS; i++) + marshal_frame(cpu, i); + per_cpu(last_timestamp, cpu) = 0; + } + printed_monotonic_warning = false; + + /* Capture the start time */ + gator_summary(); + + /* start all events */ + list_for_each_entry(gi, &gator_events, list) { + if (gi->start && gi->start() != 0) { + struct list_head *ptr = gi->list.prev; + + while (ptr != &gator_events) { + gi = list_entry(ptr, struct gator_interface, list); + + if (gi->stop) + gi->stop(); + + ptr = ptr->prev; + } + goto events_failure; + } + } + + /* cookies shall be initialized before trace_sched_start() and gator_timer_start() */ + if (cookies_initialize()) + goto cookies_failure; + if (gator_annotate_start()) + goto annotate_failure; + if (gator_trace_sched_start()) + goto sched_failure; + if (gator_trace_power_start()) + goto power_failure; + if (gator_trace_gpu_start()) + goto gpu_failure; + if (gator_timer_start(gator_timer_count)) + goto timer_failure; + if (gator_notifier_start()) + goto notifier_failure; + + return 0; + +notifier_failure: + gator_timer_stop(); +timer_failure: + gator_trace_gpu_stop(); +gpu_failure: + gator_trace_power_stop(); +power_failure: + gator_trace_sched_stop(); +sched_failure: + gator_annotate_stop(); +annotate_failure: + cookies_release(); +cookies_failure: + /* stop all events */ + list_for_each_entry(gi, &gator_events, list) + if (gi->stop) + gi->stop(); +events_failure: + gator_migrate_stop(); +migrate_failure: + gator_buffer_wake_run = false; + up(&gator_buffer_wake_sem); + gator_buffer_wake_thread = NULL; +bwake_failure: + + return -1; +} + +static void gator_stop(void) +{ + struct gator_interface *gi; + + gator_annotate_stop(); + gator_trace_sched_stop(); + gator_trace_power_stop(); + gator_trace_gpu_stop(); + + /* stop all interrupt callback reads before tearing down other interfaces */ + gator_notifier_stop(); /* should be called before gator_timer_stop to avoid re-enabling the hrtimer after it has been offlined */ + gator_timer_stop(); + + /* stop all events */ + list_for_each_entry(gi, &gator_events, list) + if (gi->stop) + gi->stop(); + + gator_migrate_stop(); + + gator_buffer_wake_run = false; + up(&gator_buffer_wake_sem); + gator_buffer_wake_thread = NULL; +} + +/****************************************************************************** + * Filesystem + ******************************************************************************/ +/* fopen("buffer") */ +static int gator_op_setup(void) +{ + int err = 0; + int cpu, i; + + mutex_lock(&start_mutex); + + gator_buffer_size[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE; + gator_buffer_mask[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE - 1; + + gator_buffer_size[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE; + gator_buffer_mask[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE - 1; + + gator_buffer_size[NAME_BUF] = NAME_BUFFER_SIZE; + gator_buffer_mask[NAME_BUF] = NAME_BUFFER_SIZE - 1; + + gator_buffer_size[COUNTER_BUF] = COUNTER_BUFFER_SIZE; + gator_buffer_mask[COUNTER_BUF] = COUNTER_BUFFER_SIZE - 1; + + gator_buffer_size[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE; + gator_buffer_mask[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE - 1; + + gator_buffer_size[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE; + gator_buffer_mask[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE - 1; + + gator_buffer_size[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE; + gator_buffer_mask[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE - 1; + + gator_buffer_size[IDLE_BUF] = IDLE_BUFFER_SIZE; + gator_buffer_mask[IDLE_BUF] = IDLE_BUFFER_SIZE - 1; + + gator_buffer_size[ACTIVITY_BUF] = ACTIVITY_BUFFER_SIZE; + gator_buffer_mask[ACTIVITY_BUF] = ACTIVITY_BUFFER_SIZE - 1; + + /* Initialize percpu per buffer variables */ + for (i = 0; i < NUM_GATOR_BUFS; i++) { + /* Verify buffers are a power of 2 */ + if (gator_buffer_size[i] & (gator_buffer_size[i] - 1)) { + err = -ENOEXEC; + goto setup_error; + } + + for_each_present_cpu(cpu) { + per_cpu(gator_buffer_read, cpu)[i] = 0; + per_cpu(gator_buffer_write, cpu)[i] = 0; + per_cpu(gator_buffer_commit, cpu)[i] = 0; + per_cpu(buffer_space_available, cpu)[i] = true; + per_cpu(gator_buffer_commit_time, cpu) = gator_live_rate; + + /* Annotation is a special case that only uses a single buffer */ + if (cpu > 0 && i == ANNOTATE_BUF) { + per_cpu(gator_buffer, cpu)[i] = NULL; + continue; + } + + per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]); + if (!per_cpu(gator_buffer, cpu)[i]) { + err = -ENOMEM; + goto setup_error; + } + } + } + +setup_error: + mutex_unlock(&start_mutex); + return err; +} + +/* Actually start profiling (echo 1>/dev/gator/enable) */ +static int gator_op_start(void) +{ + int err = 0; + + mutex_lock(&start_mutex); + + if (gator_started || gator_start()) + err = -EINVAL; + else + gator_started = 1; + + mutex_unlock(&start_mutex); + + return err; +} + +/* echo 0>/dev/gator/enable */ +static void gator_op_stop(void) +{ + mutex_lock(&start_mutex); + + if (gator_started) { + gator_stop(); + + mutex_lock(&gator_buffer_mutex); + + gator_started = 0; + gator_monotonic_started = 0; + cookies_release(); + wake_up(&gator_buffer_wait); + + mutex_unlock(&gator_buffer_mutex); + } + + mutex_unlock(&start_mutex); +} + +static void gator_shutdown(void) +{ + int cpu, i; + + mutex_lock(&start_mutex); + + for_each_present_cpu(cpu) { + mutex_lock(&gator_buffer_mutex); + for (i = 0; i < NUM_GATOR_BUFS; i++) { + vfree(per_cpu(gator_buffer, cpu)[i]); + per_cpu(gator_buffer, cpu)[i] = NULL; + per_cpu(gator_buffer_read, cpu)[i] = 0; + per_cpu(gator_buffer_write, cpu)[i] = 0; + per_cpu(gator_buffer_commit, cpu)[i] = 0; + per_cpu(buffer_space_available, cpu)[i] = true; + per_cpu(gator_buffer_commit_time, cpu) = 0; + } + mutex_unlock(&gator_buffer_mutex); + } + + memset(&sent_core_name, 0, sizeof(sent_core_name)); + + mutex_unlock(&start_mutex); +} + +static int gator_set_backtrace(unsigned long val) +{ + int err = 0; + + mutex_lock(&start_mutex); + + if (gator_started) + err = -EBUSY; + else + gator_backtrace_depth = val; + + mutex_unlock(&start_mutex); + + return err; +} + +static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset) +{ + return gatorfs_ulong_to_user(gator_started, buf, count, offset); +} + +static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset) +{ + unsigned long val = 0; + int retval = 0; + + if (*offset) + return -EINVAL; + + retval = gatorfs_ulong_from_user(&val, buf, count); + if (retval) + return retval; + + if (val) + retval = gator_op_start(); + else + gator_op_stop(); + + if (retval) + return retval; + return count; +} + +static const struct file_operations enable_fops = { + .read = enable_read, + .write = enable_write, +}; + +static int userspace_buffer_open(struct inode *inode, struct file *file) +{ + int err = -EPERM; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (test_and_set_bit_lock(0, &gator_buffer_opened)) + return -EBUSY; + + err = gator_op_setup(); + if (err) + goto fail; + + /* NB: the actual start happens from userspace + * echo 1 >/dev/gator/enable + */ + + return 0; + +fail: + __clear_bit_unlock(0, &gator_buffer_opened); + return err; +} + +static int userspace_buffer_release(struct inode *inode, struct file *file) +{ + gator_op_stop(); + gator_shutdown(); + __clear_bit_unlock(0, &gator_buffer_opened); + return 0; +} + +static ssize_t userspace_buffer_read(struct file *file, char __user *buf, size_t count, loff_t *offset) +{ + int commit, length1, length2, read; + char *buffer1; + char *buffer2; + int cpu, buftype; + int written = 0; + + /* ensure there is enough space for a whole frame */ + if (count < userspace_buffer_size || *offset) + return -EINVAL; + + /* sleep until the condition is true or a signal is received the + * condition is checked each time gator_buffer_wait is woken up + */ + wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(-1, -1, &cpu, &buftype) || !gator_started); + + if (signal_pending(current)) + return -EINTR; + + if (buftype == -1 || cpu == -1) + return 0; + + mutex_lock(&gator_buffer_mutex); + + do { + read = per_cpu(gator_buffer_read, cpu)[buftype]; + commit = per_cpu(gator_buffer_commit, cpu)[buftype]; + + /* May happen if the buffer is freed during pending reads. */ + if (!per_cpu(gator_buffer, cpu)[buftype]) + break; + + /* determine the size of two halves */ + length1 = commit - read; + length2 = 0; + buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]); + buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]); + if (length1 < 0) { + length1 = gator_buffer_size[buftype] - read; + length2 = commit; + } + + if (length1 + length2 > count - written) + break; + + /* start, middle or end */ + if (length1 > 0 && copy_to_user(&buf[written], buffer1, length1)) + break; + + /* possible wrap around */ + if (length2 > 0 && copy_to_user(&buf[written + length1], buffer2, length2)) + break; + + per_cpu(gator_buffer_read, cpu)[buftype] = commit; + written += length1 + length2; + + /* Wake up annotate_write if more space is available */ + if (buftype == ANNOTATE_BUF) + wake_up(&gator_annotate_wait); + } while (buffer_commit_ready(cpu, buftype, &cpu, &buftype)); + + mutex_unlock(&gator_buffer_mutex); + + /* kick just in case we've lost an SMP event */ + wake_up(&gator_buffer_wait); + + return written > 0 ? written : -EFAULT; +} + +static const struct file_operations gator_event_buffer_fops = { + .open = userspace_buffer_open, + .release = userspace_buffer_release, + .read = userspace_buffer_read, +}; + +static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset) +{ + return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count, offset); +} + +static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset) +{ + unsigned long val; + int retval; + + if (*offset) + return -EINVAL; + + retval = gatorfs_ulong_from_user(&val, buf, count); + if (retval) + return retval; + + retval = gator_set_backtrace(val); + + if (retval) + return retval; + return count; +} + +static const struct file_operations depth_fops = { + .read = depth_read, + .write = depth_write +}; + +static void gator_op_create_files(struct super_block *sb, struct dentry *root) +{ + struct dentry *dir; + int cpu; + + /* reinitialize default values */ + gator_cpu_cores = 0; + for_each_present_cpu(cpu) { + gator_cpu_cores++; + } + userspace_buffer_size = BACKTRACE_BUFFER_SIZE; + gator_response_type = 1; + gator_live_rate = 0; + + gatorfs_create_file(sb, root, "enable", &enable_fops); + gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops); + gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops); + gatorfs_create_ro_ulong(sb, root, "cpu_cores", &gator_cpu_cores); + gatorfs_create_ro_ulong(sb, root, "buffer_size", &userspace_buffer_size); + gatorfs_create_ulong(sb, root, "tick", &gator_timer_count); + gatorfs_create_ulong(sb, root, "response_type", &gator_response_type); + gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version); + gatorfs_create_ro_u64(sb, root, "started", &gator_monotonic_started); + gatorfs_create_u64(sb, root, "live_rate", &gator_live_rate); + + gator_annotate_create_files(sb, root); + + /* Linux Events */ + dir = gatorfs_mkdir(sb, root, "events"); + gator_pmu_create_files(sb, root, dir); +} + +/****************************************************************************** + * Module + ******************************************************************************/ + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 15, 0) + +#define GATOR_TRACEPOINTS \ + GATOR_HANDLE_TRACEPOINT(block_rq_complete); \ + GATOR_HANDLE_TRACEPOINT(cpu_frequency); \ + GATOR_HANDLE_TRACEPOINT(cpu_idle); \ + GATOR_HANDLE_TRACEPOINT(cpu_migrate_begin); \ + GATOR_HANDLE_TRACEPOINT(cpu_migrate_current); \ + GATOR_HANDLE_TRACEPOINT(cpu_migrate_finish); \ + GATOR_HANDLE_TRACEPOINT(irq_handler_exit); \ + GATOR_HANDLE_TRACEPOINT(mali_hw_counter); \ + GATOR_HANDLE_TRACEPOINT(mali_job_slots_event); \ + GATOR_HANDLE_TRACEPOINT(mali_mmu_as_in_use); \ + GATOR_HANDLE_TRACEPOINT(mali_mmu_as_released); \ + GATOR_HANDLE_TRACEPOINT(mali_page_fault_insert_pages); \ + GATOR_HANDLE_TRACEPOINT(mali_pm_status); \ + GATOR_HANDLE_TRACEPOINT(mali_sw_counter); \ + GATOR_HANDLE_TRACEPOINT(mali_sw_counters); \ + GATOR_HANDLE_TRACEPOINT(mali_timeline_event); \ + GATOR_HANDLE_TRACEPOINT(mali_total_alloc_pages_change); \ + GATOR_HANDLE_TRACEPOINT(mm_page_alloc); \ + GATOR_HANDLE_TRACEPOINT(mm_page_free); \ + GATOR_HANDLE_TRACEPOINT(mm_page_free_batched); \ + GATOR_HANDLE_TRACEPOINT(sched_process_exec); \ + GATOR_HANDLE_TRACEPOINT(sched_process_fork); \ + GATOR_HANDLE_TRACEPOINT(sched_process_free); \ + GATOR_HANDLE_TRACEPOINT(sched_switch); \ + GATOR_HANDLE_TRACEPOINT(softirq_exit); \ + GATOR_HANDLE_TRACEPOINT(task_rename); \ + +#define GATOR_HANDLE_TRACEPOINT(probe_name) \ + struct tracepoint *gator_tracepoint_##probe_name +GATOR_TRACEPOINTS; +#undef GATOR_HANDLE_TRACEPOINT + +static void gator_save_tracepoint(struct tracepoint *tp, void *priv) +{ + pr_debug("gator: gator_save_tracepoint(%s)\n", tp->name); +#define GATOR_HANDLE_TRACEPOINT(probe_name) \ + do { \ + if (strcmp(tp->name, #probe_name) == 0) { \ + gator_tracepoint_##probe_name = tp; \ + return; \ + } \ + } while (0) +GATOR_TRACEPOINTS; +#undef GATOR_HANDLE_TRACEPOINT +} + +#if defined(CONFIG_MODULES) + +static void gator_unsave_tracepoint(struct tracepoint *tp, void *priv) +{ + pr_debug("gator: gator_unsave_tracepoint(%s)\n", tp->name); +#define GATOR_HANDLE_TRACEPOINT(probe_name) \ + do { \ + if (strcmp(tp->name, #probe_name) == 0) { \ + gator_tracepoint_##probe_name = NULL; \ + return; \ + } \ + } while (0) +GATOR_TRACEPOINTS; +#undef GATOR_HANDLE_TRACEPOINT +} + +int gator_new_tracepoint_module(struct notifier_block * nb, unsigned long action, void * data) +{ + struct tp_module * tp_mod = (struct tp_module *) data; + struct tracepoint * const * begin = tp_mod->mod->tracepoints_ptrs; + struct tracepoint * const * end = tp_mod->mod->tracepoints_ptrs + tp_mod->mod->num_tracepoints; + + pr_debug("gator: new tracepoint module registered %s\n", tp_mod->mod->name); + + if (action == MODULE_STATE_COMING) + { + for (; begin != end; ++begin) { + gator_save_tracepoint(*begin, NULL); + } + } + else if (action == MODULE_STATE_GOING) + { + for (; begin != end; ++begin) { + gator_unsave_tracepoint(*begin, NULL); + } + } + else + { + pr_debug("gator: unexpected action value in gator_new_tracepoint_module: 0x%lx\n", action); + } + + return 0; +} + +static struct notifier_block tracepoint_notifier_block = { + .notifier_call = gator_new_tracepoint_module +}; + +#endif + +#endif + + +/* The user may define 'CONFIG_GATOR_DO_NOT_ONLINE_CORES_AT_STARTUP' to prevent offline cores from being + * started when gator is loaded, or alternatively they can set 'disable_cpu_onlining' when module is loaded */ +#if defined(CONFIG_SMP) && !defined(CONFIG_GATOR_DO_NOT_ONLINE_CORES_AT_STARTUP) +MODULE_PARM_DESC(disable_cpu_onlining, "Do not online all cores when module starts"); +static bool disable_cpu_onlining; +module_param(disable_cpu_onlining, bool, 0644); +#endif + +static int __init gator_module_init(void) +{ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 15, 0) + /* scan kernel built in tracepoints */ + for_each_kernel_tracepoint(gator_save_tracepoint, NULL); +#if defined(CONFIG_MODULES) + /* register for notification of new tracepoint modules */ + register_tracepoint_module_notifier(&tracepoint_notifier_block); +#endif +#endif + + if (gatorfs_register()) + return -1; + + if (gator_init()) { + gatorfs_unregister(); + return -1; + } + + setup_deferrable_timer_on_stack(&gator_buffer_wake_up_timer, gator_buffer_wake_up, 0); + +#if defined(CONFIG_SMP) && !defined(CONFIG_GATOR_DO_NOT_ONLINE_CORES_AT_STARTUP) + /* Online all cores */ + if (!disable_cpu_onlining) { + int cpu; + for_each_present_cpu(cpu) { + if (!cpu_online(cpu)) { + pr_notice("gator: Onlining cpu %i\n", cpu); + cpu_up(cpu); + } + } + } +#endif + + /* Initialize the list of cpuids */ + memset(gator_cpuids, -1, sizeof(gator_cpuids)); + on_each_cpu(gator_read_cpuid, NULL, 1); + + return 0; +} + +static void __exit gator_module_exit(void) +{ +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 15, 0)) && defined(CONFIG_MODULES) + /* unregister for notification of new tracepoint modules */ + unregister_tracepoint_module_notifier(&tracepoint_notifier_block); +#endif + + del_timer_sync(&gator_buffer_wake_up_timer); + tracepoint_synchronize_unregister(); + gator_exit(); + gatorfs_unregister(); + gator_pmu_exit(); +} + +module_init(gator_module_init); +module_exit(gator_module_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Arm Ltd"); +MODULE_DESCRIPTION("Gator system profiler"); +#define STRIFY2(ARG) #ARG +#define STRIFY(ARG) STRIFY2(ARG) +MODULE_VERSION(STRIFY(PROTOCOL_VERSION)); diff --git a/drivers/gator/gator_marshaling.c b/drivers/gator/gator_marshaling.c new file mode 100644 index 000000000000..b395514a8203 --- /dev/null +++ b/drivers/gator/gator_marshaling.c @@ -0,0 +1,379 @@ +/** + * Copyright (C) Arm Limited 2012-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#define NEWLINE_CANARY \ + /* Unix */ \ + "1\n" \ + /* Windows */ \ + "2\r\n" \ + /* Mac OS */ \ + "3\r" \ + /* RISC OS */ \ + "4\n\r" \ + /* Add another character so the length isn't 0x0a bytes */ \ + "5" + +#ifdef MALI_SUPPORT +#include "gator_events_mali_common.h" +#endif + +static void marshal_summary(long long timestamp, long long uptime, long long monotonic_delta, const char *uname) +{ + unsigned long flags; + int cpu = 0; + char buf[32]; + + local_irq_save(flags); + gator_buffer_write_packed_int(cpu, SUMMARY_BUF, MESSAGE_SUMMARY); + gator_buffer_write_string(cpu, SUMMARY_BUF, NEWLINE_CANARY); + gator_buffer_write_packed_int64(cpu, SUMMARY_BUF, timestamp); + gator_buffer_write_packed_int64(cpu, SUMMARY_BUF, uptime); + gator_buffer_write_packed_int64(cpu, SUMMARY_BUF, monotonic_delta); + gator_buffer_write_string(cpu, SUMMARY_BUF, "uname"); + gator_buffer_write_string(cpu, SUMMARY_BUF, uname); + gator_buffer_write_string(cpu, SUMMARY_BUF, "PAGESIZE"); + snprintf(buf, sizeof(buf), "%lu", PAGE_SIZE); + gator_buffer_write_string(cpu, SUMMARY_BUF, buf); +#if GATOR_IKS_SUPPORT + gator_buffer_write_string(cpu, SUMMARY_BUF, "iks"); + gator_buffer_write_string(cpu, SUMMARY_BUF, ""); +#endif +#ifdef CONFIG_PREEMPT_RTB + gator_buffer_write_string(cpu, SUMMARY_BUF, "preempt_rtb"); + gator_buffer_write_string(cpu, SUMMARY_BUF, ""); +#endif +#ifdef CONFIG_PREEMPT_RT_FULL + gator_buffer_write_string(cpu, SUMMARY_BUF, "preempt_rt_full"); + gator_buffer_write_string(cpu, SUMMARY_BUF, ""); +#endif + /* Let Streamline know which GPU is used so that it can label the GPU Activity appropriately. This is a temporary fix, to be improved in a future release. */ +#ifdef MALI_SUPPORT + gator_buffer_write_string(cpu, SUMMARY_BUF, "mali_type"); +#if (MALI_SUPPORT == MALI_4xx) + gator_buffer_write_string(cpu, SUMMARY_BUF, "4xx"); +#elif (MALI_SUPPORT == MALI_MIDGARD_OR_BIFROST) + gator_buffer_write_string(cpu, SUMMARY_BUF, "6xx"); +#else + gator_buffer_write_string(cpu, SUMMARY_BUF, "unknown"); +#endif +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 2, 0) + gator_buffer_write_string(cpu, SUMMARY_BUF, "nosync"); + gator_buffer_write_string(cpu, SUMMARY_BUF, ""); +#endif + gator_buffer_write_string(cpu, SUMMARY_BUF, ""); + /* Commit the buffer now so it can be one of the first frames read by Streamline */ + local_irq_restore(flags); + gator_commit_buffer(cpu, SUMMARY_BUF, gator_get_time()); +} + +static bool marshal_cookie_header(const char *text) +{ + int cpu = get_physical_cpu(); + + return buffer_check_space(cpu, NAME_BUF, strlen(text) + 3 * MAXSIZE_PACK32); +} + +static void marshal_cookie(int cookie, const char *text) +{ + int cpu = get_physical_cpu(); + /* buffer_check_space already called by marshal_cookie_header */ + gator_buffer_write_packed_int(cpu, NAME_BUF, MESSAGE_COOKIE); + gator_buffer_write_packed_int(cpu, NAME_BUF, cookie); + gator_buffer_write_string(cpu, NAME_BUF, text); + buffer_check(cpu, NAME_BUF, gator_get_time()); +} + +static void marshal_thread_name(int pid, char *name) +{ + unsigned long flags, cpu; + u64 time; + + local_irq_save(flags); + cpu = get_physical_cpu(); + time = gator_get_time(); + if (buffer_check_space(cpu, NAME_BUF, TASK_COMM_LEN + 3 * MAXSIZE_PACK32 + MAXSIZE_PACK64)) { + gator_buffer_write_packed_int(cpu, NAME_BUF, MESSAGE_THREAD_NAME); + gator_buffer_write_packed_int64(cpu, NAME_BUF, time); + gator_buffer_write_packed_int(cpu, NAME_BUF, pid); + gator_buffer_write_string(cpu, NAME_BUF, name); + } + local_irq_restore(flags); + buffer_check(cpu, NAME_BUF, time); +} + +static void marshal_link(int cookie, int tgid, int pid) +{ + unsigned long cpu = get_physical_cpu(), flags; + u64 time; + + local_irq_save(flags); + time = gator_get_time(); + if (buffer_check_space(cpu, ACTIVITY_BUF, MAXSIZE_PACK64 + 5 * MAXSIZE_PACK32)) { + gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, MESSAGE_LINK); + gator_buffer_write_packed_int64(cpu, ACTIVITY_BUF, time); + gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, cookie); + gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, tgid); + gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, pid); + } + local_irq_restore(flags); + /* Check and commit; commit is set to occur once buffer is 3/4 full */ + buffer_check(cpu, ACTIVITY_BUF, time); +} + +static bool marshal_backtrace_header(int exec_cookie, int tgid, int pid, u64 time) +{ + int cpu = get_physical_cpu(); + + if (!buffer_check_space(cpu, BACKTRACE_BUF, MAXSIZE_PACK64 + 5 * MAXSIZE_PACK32 + gator_backtrace_depth * 2 * MAXSIZE_PACK32)) { + /* Check and commit; commit is set to occur once buffer is 3/4 full */ + buffer_check(cpu, BACKTRACE_BUF, time); + + return false; + } + + gator_buffer_write_packed_int64(cpu, BACKTRACE_BUF, time); + gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, exec_cookie); + gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, tgid); + gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, pid); + + return true; +} + +static void marshal_backtrace(unsigned long address, int cookie, int in_kernel) +{ + int cpu = get_physical_cpu(); + + if (cookie == 0 && !in_kernel) + cookie = UNRESOLVED_COOKIE; + gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, cookie); + gator_buffer_write_packed_int64(cpu, BACKTRACE_BUF, address); +} + +static void marshal_backtrace_footer(u64 time) +{ + int cpu = get_physical_cpu(); + + gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, MESSAGE_END_BACKTRACE); + + /* Check and commit; commit is set to occur once buffer is 3/4 full */ + buffer_check(cpu, BACKTRACE_BUF, time); +} + +static bool marshal_event_header(u64 time) +{ + unsigned long flags, cpu = get_physical_cpu(); + bool retval = false; + + local_irq_save(flags); + if (buffer_check_space(cpu, BLOCK_COUNTER_BUF, MAXSIZE_PACK32 + MAXSIZE_PACK64)) { + gator_buffer_write_packed_int(cpu, BLOCK_COUNTER_BUF, 0); /* key of zero indicates a timestamp */ + gator_buffer_write_packed_int64(cpu, BLOCK_COUNTER_BUF, time); + retval = true; + } + local_irq_restore(flags); + + return retval; +} + +static void marshal_event(int len, int *buffer) +{ + unsigned long i, flags, cpu = get_physical_cpu(); + + if (len <= 0) + return; + + /* length must be even since all data is a (key, value) pair */ + if (len & 0x1) { + pr_err("gator: invalid counter data detected and discarded\n"); + return; + } + + /* events must be written in key,value pairs */ + local_irq_save(flags); + for (i = 0; i < len; i += 2) { + if (!buffer_check_space(cpu, BLOCK_COUNTER_BUF, 2 * MAXSIZE_PACK32)) + break; + gator_buffer_write_packed_int(cpu, BLOCK_COUNTER_BUF, buffer[i]); + gator_buffer_write_packed_int(cpu, BLOCK_COUNTER_BUF, buffer[i + 1]); + } + local_irq_restore(flags); +} + +static void marshal_event64(int len, long long *buffer64) +{ + unsigned long i, flags, cpu = get_physical_cpu(); + + if (len <= 0) + return; + + /* length must be even since all data is a (key, value) pair */ + if (len & 0x1) { + pr_err("gator: invalid counter data detected and discarded\n"); + return; + } + + /* events must be written in key,value pairs */ + local_irq_save(flags); + for (i = 0; i < len; i += 2) { + if (!buffer_check_space(cpu, BLOCK_COUNTER_BUF, 2 * MAXSIZE_PACK64)) + break; + gator_buffer_write_packed_int64(cpu, BLOCK_COUNTER_BUF, buffer64[i]); + gator_buffer_write_packed_int64(cpu, BLOCK_COUNTER_BUF, buffer64[i + 1]); + } + local_irq_restore(flags); +} + +static void __maybe_unused marshal_event_single(int core, int key, int value) +{ + unsigned long flags, cpu; + u64 time; + + local_irq_save(flags); + cpu = get_physical_cpu(); + time = gator_get_time(); + if (buffer_check_space(cpu, COUNTER_BUF, MAXSIZE_PACK64 + 3 * MAXSIZE_PACK32)) { + gator_buffer_write_packed_int64(cpu, COUNTER_BUF, time); + gator_buffer_write_packed_int(cpu, COUNTER_BUF, core); + gator_buffer_write_packed_int(cpu, COUNTER_BUF, key); + gator_buffer_write_packed_int(cpu, COUNTER_BUF, value); + } + local_irq_restore(flags); + /* Check and commit; commit is set to occur once buffer is 3/4 full */ + buffer_check(cpu, COUNTER_BUF, time); +} + +static void __maybe_unused marshal_event_single64(int core, int key, long long value) +{ + unsigned long flags, cpu; + u64 time; + + local_irq_save(flags); + cpu = get_physical_cpu(); + time = gator_get_time(); + if (buffer_check_space(cpu, COUNTER_BUF, 2 * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) { + gator_buffer_write_packed_int64(cpu, COUNTER_BUF, time); + gator_buffer_write_packed_int(cpu, COUNTER_BUF, core); + gator_buffer_write_packed_int(cpu, COUNTER_BUF, key); + gator_buffer_write_packed_int64(cpu, COUNTER_BUF, value); + } + local_irq_restore(flags); + /* Check and commit; commit is set to occur once buffer is 3/4 full */ + buffer_check(cpu, COUNTER_BUF, time); +} + +static void marshal_sched_trace_switch(int pid, int state) +{ + unsigned long cpu = get_physical_cpu(), flags; + u64 time; + + if (!per_cpu(gator_buffer, cpu)[SCHED_TRACE_BUF]) + return; + + local_irq_save(flags); + time = gator_get_time(); + if (buffer_check_space(cpu, SCHED_TRACE_BUF, MAXSIZE_PACK64 + 5 * MAXSIZE_PACK32)) { + gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, MESSAGE_SCHED_SWITCH); + gator_buffer_write_packed_int64(cpu, SCHED_TRACE_BUF, time); + gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, pid); + gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, state); + } + local_irq_restore(flags); + /* Check and commit; commit is set to occur once buffer is 3/4 full */ + buffer_check(cpu, SCHED_TRACE_BUF, time); +} + +static void marshal_sched_trace_exit(int tgid, int pid) +{ + unsigned long cpu = get_physical_cpu(), flags; + u64 time; + + if (!per_cpu(gator_buffer, cpu)[SCHED_TRACE_BUF]) + return; + + local_irq_save(flags); + time = gator_get_time(); + if (buffer_check_space(cpu, SCHED_TRACE_BUF, MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) { + gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, MESSAGE_SCHED_EXIT); + gator_buffer_write_packed_int64(cpu, SCHED_TRACE_BUF, time); + gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, pid); + } + local_irq_restore(flags); + /* Check and commit; commit is set to occur once buffer is 3/4 full */ + buffer_check(cpu, SCHED_TRACE_BUF, time); +} + +#if GATOR_CPU_FREQ_SUPPORT +static void marshal_idle(int core, int state) +{ + unsigned long flags, cpu; + u64 time; + + local_irq_save(flags); + cpu = get_physical_cpu(); + time = gator_get_time(); + if (buffer_check_space(cpu, IDLE_BUF, MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) { + gator_buffer_write_packed_int(cpu, IDLE_BUF, state); + gator_buffer_write_packed_int64(cpu, IDLE_BUF, time); + gator_buffer_write_packed_int(cpu, IDLE_BUF, core); + } + local_irq_restore(flags); + /* Check and commit; commit is set to occur once buffer is 3/4 full */ + buffer_check(cpu, IDLE_BUF, time); +} +#endif + +#if defined(__arm__) || defined(__aarch64__) +static void marshal_core_name(const int core, const int cpuid, const char *name) +{ + int cpu = get_physical_cpu(); + unsigned long flags; + + local_irq_save(flags); + if (buffer_check_space(cpu, SUMMARY_BUF, MAXSIZE_PACK32 + MAXSIZE_CORE_NAME)) { + gator_buffer_write_packed_int(cpu, SUMMARY_BUF, MESSAGE_CORE_NAME); + gator_buffer_write_packed_int(cpu, SUMMARY_BUF, core); + gator_buffer_write_packed_int(cpu, SUMMARY_BUF, cpuid); + gator_buffer_write_string(cpu, SUMMARY_BUF, name); + } + /* Commit core names now so that they can show up in live */ + local_irq_restore(flags); + gator_commit_buffer(cpu, SUMMARY_BUF, gator_get_time()); +} +#endif + +static void marshal_activity_switch(int core, int key, int activity, int pid, int state) +{ + unsigned long cpu = get_physical_cpu(), flags; + u64 time; + + if (!per_cpu(gator_buffer, cpu)[ACTIVITY_BUF]) + return; + + local_irq_save(flags); + time = gator_get_time(); + if (buffer_check_space(cpu, ACTIVITY_BUF, MAXSIZE_PACK64 + 5 * MAXSIZE_PACK32)) { + gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, MESSAGE_SWITCH); + gator_buffer_write_packed_int64(cpu, ACTIVITY_BUF, time); + gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, core); + gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, key); + gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, activity); + gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, pid); + gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, state); + } + local_irq_restore(flags); + /* Check and commit; commit is set to occur once buffer is 3/4 full */ + buffer_check(cpu, ACTIVITY_BUF, time); +} + +void gator_marshal_activity_switch(int core, int key, int activity, int pid) +{ + /* state is reserved for cpu use only */ + marshal_activity_switch(core, key, activity, pid, 0); +} diff --git a/drivers/gator/gator_pmu.c b/drivers/gator/gator_pmu.c new file mode 100644 index 000000000000..e4499265ca8b --- /dev/null +++ b/drivers/gator/gator_pmu.c @@ -0,0 +1,379 @@ +/** + * Copyright (C) Arm Limited 2015-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include + +#ifdef CONFIG_64BIT +# define GATOR_ATOMIC_T atomic64_t +# define GATOR_ATOMIC_CMPXCHG atomic64_cmpxchg +# define GATOR_ATOMIC_READ atomic64_read +#else +# define GATOR_ATOMIC_T atomic_t +# define GATOR_ATOMIC_CMPXCHG atomic_cmpxchg +# define GATOR_ATOMIC_READ atomic_read +#endif + +#define GATOR_ATOMIC_PTR_TO_COUNTER(ptr) ((unsigned long) (ptr)) + +struct uncore_pmu { + struct list_head list; + unsigned long pmnc_counters; + unsigned long has_cycles_counter; + /* Perf PMU name */ + char pmnc_name[MAXSIZE_CORE_NAME]; + /* gatorfs event name */ + char core_name[MAXSIZE_CORE_NAME]; + /* cpumask - actually an atomic so we can CAS the pointer once without lock*/ + GATOR_ATOMIC_T cpumask_atomic; +}; + +static LIST_HEAD(uncore_pmus); +static LIST_HEAD(gator_cpus); +static DEFINE_MUTEX(pmu_mutex); + +static struct super_block *gator_sb; +static struct dentry *gator_events_dir; + +static const struct gator_cpu gator_pmu_other = { + .pmnc_name = "Other", + .cpuid = 0xfffff, + .core_name = "Other", + .pmnc_counters = 6, +}; + +const struct gator_cpu *gator_clusters[GATOR_CLUSTER_COUNT]; +int gator_cluster_count; +static ulong gator_cluster_ids[GATOR_CLUSTER_COUNT]; + +static const struct gator_cpu *gator_find_cpu_by_cpuid(const u32 cpuid) +{ + const struct gator_cpu *gator_cpu; + + list_for_each_entry(gator_cpu, &gator_cpus, list) { + if (gator_cpu->cpuid == cpuid) + return gator_cpu; + } + + return NULL; +} + +static const char OLD_PMU_PREFIX[] = "ARMv7 Cortex-"; +static const char NEW_PMU_PREFIX[] = "ARMv7_Cortex_"; + +__maybe_unused +static const struct gator_cpu *gator_find_cpu_by_pmu_name(const char *const name) +{ + const struct gator_cpu *gator_cpu; + + list_for_each_entry(gator_cpu, &gator_cpus, list) { + if (gator_cpu->pmnc_name != NULL && + /* Do the names match exactly? */ + (strcasecmp(gator_cpu->pmnc_name, name) == 0 || + /* Do these names match but have the old vs new prefix? */ + ((strncasecmp(name, OLD_PMU_PREFIX, sizeof(OLD_PMU_PREFIX) - 1) == 0 && + strncasecmp(gator_cpu->pmnc_name, NEW_PMU_PREFIX, sizeof(NEW_PMU_PREFIX) - 1) == 0 && + strcasecmp(name + sizeof(OLD_PMU_PREFIX) - 1, gator_cpu->pmnc_name + sizeof(NEW_PMU_PREFIX) - 1) == 0)))) + return gator_cpu; + } + + return NULL; +} + +__maybe_unused +static const struct uncore_pmu *gator_find_uncore_pmu(const char *const name) +{ + const struct uncore_pmu *uncore_pmu; + + list_for_each_entry(uncore_pmu, &uncore_pmus, list) { + if (uncore_pmu->pmnc_name != NULL && strcasecmp(uncore_pmu->pmnc_name, name) == 0) + return uncore_pmu; + } + + return NULL; +} + +static ssize_t gator_pmu_init_write(struct file *file, char const __user *buf, size_t count, loff_t *offset) +{ + struct gator_interface *gi; + int i; + + if (gator_events_perf_pmu_reread() != 0 || + gator_events_perf_pmu_create_files(gator_sb, gator_events_dir) != 0) + return -EINVAL; + + if (gator_cluster_count == 0) + gator_clusters[gator_cluster_count++] = &gator_pmu_other; + + /* cluster information */ + { + struct dentry *dir; + + dir = gatorfs_mkdir(gator_sb, file->f_path.dentry->d_parent, "clusters"); + for (i = 0; i < gator_cluster_count; i++) { + gator_cluster_ids[i] = i; + gatorfs_create_ro_ulong(gator_sb, dir, gator_clusters[i]->pmnc_name, &gator_cluster_ids[i]); + } + } + + /* needs PMU info, so initialize afterwards */ + gator_trace_power_init(); + if (gator_trace_power_create_files(gator_sb, gator_events_dir) != 0) + return -EINVAL; + + gator_trace_sched_init(); + if (sched_trace_create_files(gator_sb, gator_events_dir) != 0) + return -EINVAL; + + /* events sources */ + for (i = 0; i < ARRAY_SIZE(gator_events_list); i++) + if (gator_events_list[i]) + gator_events_list[i](); + + list_for_each_entry(gi, &gator_events, list) + if (gi->create_files) + if (gi->create_files(gator_sb, gator_events_dir) != 0) + pr_err("gator: create_files failed for %s\n", gi->name); + + return count; +} + +static const struct file_operations gator_pmu_init_fops = { + .write = gator_pmu_init_write, +}; + +static ssize_t gator_pmu_str_read_file(struct file *file, char __user *buf, size_t count, loff_t *offset) +{ + char *const val = file->private_data; + + return simple_read_from_buffer(buf, count, offset, val, strlen(val)); +} + +static ssize_t gator_pmu_str_write_file(struct file *file, char const __user *buf, size_t count, loff_t *offset) +{ + char *value = file->private_data; + + if (*offset) + return -EINVAL; + + if (count >= MAXSIZE_CORE_NAME) + return -EINVAL; + if (copy_from_user(value, buf, count)) + return -EFAULT; + value[count] = 0; + value = strstrip(value); + + return count; +} + +static const struct file_operations gator_pmu_str_fops = { + .read = gator_pmu_str_read_file, + .write = gator_pmu_str_write_file, + .open = default_open, +}; + +static int gator_pmu_create_str(struct super_block *sb, struct dentry *root, char const *name, char *const val) +{ + struct dentry *d = __gatorfs_create_file(sb, root, name, &gator_pmu_str_fops, 0644); + if (!d) + return -EFAULT; + + d->d_inode->i_private = val; + return 0; +} + +#define GATOR_NONE_STRING "(none)" + +static ssize_t gator_pmu_cpumask_read(struct file *file, char __user *buf, size_t count, loff_t *offset) +{ + struct uncore_pmu * uncore_pmu = NULL; + struct cpumask * cpumask = NULL; + int cpu = 0; + int bpos = 0; + char buffer[128]; + + uncore_pmu = (struct uncore_pmu *) file->private_data; + if (!uncore_pmu) + return -EFAULT; + + cpumask = (struct cpumask *) GATOR_ATOMIC_READ(&(uncore_pmu->cpumask_atomic)); + if (!cpumask) + return simple_read_from_buffer(buf, count, offset, GATOR_NONE_STRING, strlen(GATOR_NONE_STRING)); + + for_each_cpu(cpu, cpumask) { + bpos = snprintf(buffer + bpos, sizeof(buffer) - bpos, (bpos > 0 ? ",%d" : "%d"), cpu); + if (bpos >= (sizeof(buffer) - 1)) + break; + } + + return simple_read_from_buffer(buf, count, offset, buffer, strlen(buffer)); +} + +static ssize_t gator_pmu_cpumask_write(struct file *file, char const __user *ubuf, size_t count, loff_t *offset) +{ + struct uncore_pmu * uncore_pmu = NULL; + struct cpumask * cpumask = NULL; + unsigned long value = 0; + int retval = 0; + + uncore_pmu = (struct uncore_pmu *) file->private_data; + if (!uncore_pmu) + return -EFAULT; + + /* validate args */ + if (*offset) + return -EINVAL; + + /* parse ulong */ + retval = gatorfs_ulong_from_user(&value, ubuf, count); + if (retval) + return retval; + if (value >= nr_cpu_ids) + return -EINVAL; + + /* allocate the mask if it does not already exist */ + cpumask = (struct cpumask *) GATOR_ATOMIC_READ(&(uncore_pmu->cpumask_atomic)); + if (!cpumask) { + struct cpumask * ret = NULL; + + cpumask = kzalloc(cpumask_size(), GFP_KERNEL); + if (!cpumask) + return -ENOMEM; + + ret = (struct cpumask * ) GATOR_ATOMIC_CMPXCHG(&(uncore_pmu->cpumask_atomic), GATOR_ATOMIC_PTR_TO_COUNTER(NULL), GATOR_ATOMIC_PTR_TO_COUNTER(cpumask)); + if (ret != NULL) { + /* someone else got there first, free the one we allocated and use the existing instead */ + kfree(cpumask); + cpumask = ret; + } + } + + cpumask_set_cpu(value, cpumask); + return count; +} + +static const struct file_operations cpumask_fops = { + .read = gator_pmu_cpumask_read, + .write = gator_pmu_cpumask_write, + .open = default_open, +}; + +static ssize_t gator_pmu_export_write(struct file *file, char const __user *ubuf, size_t count, loff_t *offset) +{ + struct dentry *dir; + struct dentry *parent; + char buf[MAXSIZE_CORE_NAME]; + const char *str; + + if (*offset) + return -EINVAL; + + if (count >= sizeof(buf)) + return -EINVAL; + if (copy_from_user(&buf, ubuf, count)) + return -EFAULT; + buf[count] = 0; + str = strstrip(buf); + + parent = file->f_path.dentry->d_parent; + dir = gatorfs_mkdir(gator_sb, parent, buf); + if (!dir) + return -EINVAL; + + if (strcmp("pmu", parent->d_name.name) == 0) { + struct gator_cpu *gator_cpu; + + gator_cpu = kmalloc(sizeof(*gator_cpu), GFP_KERNEL); + if (gator_cpu == NULL) + return -ENOMEM; + memset(gator_cpu, 0, sizeof(*gator_cpu)); + + gatorfs_create_ulong(gator_sb, dir, "cpuid", &gator_cpu->cpuid); + gator_pmu_create_str(gator_sb, dir, "core_name", gator_cpu->core_name); + strcpy(gator_cpu->pmnc_name, str); + gator_pmu_create_str(gator_sb, dir, "dt_name", gator_cpu->dt_name); + gatorfs_create_ulong(gator_sb, dir, "pmnc_counters", &gator_cpu->pmnc_counters); + + mutex_lock(&pmu_mutex); + list_add_tail(&gator_cpu->list, &gator_cpus); /* mutex */ + mutex_unlock(&pmu_mutex); + } else { + struct uncore_pmu *uncore_pmu; + + uncore_pmu = kmalloc(sizeof(*uncore_pmu), GFP_KERNEL); + if (uncore_pmu == NULL) + return -ENOMEM; + memset(uncore_pmu, 0, sizeof(*uncore_pmu)); + + strcpy(uncore_pmu->pmnc_name, str); + gator_pmu_create_str(gator_sb, dir, "core_name", uncore_pmu->core_name); + gatorfs_create_ulong(gator_sb, dir, "pmnc_counters", &uncore_pmu->pmnc_counters); + gatorfs_create_ulong(gator_sb, dir, "has_cycles_counter", &uncore_pmu->has_cycles_counter); + gatorfs_create_file_data(gator_sb, dir, "cpumask", &cpumask_fops, uncore_pmu); + + mutex_lock(&pmu_mutex); + list_add_tail(&uncore_pmu->list, &uncore_pmus); /* mutex */ + mutex_unlock(&pmu_mutex); + } + + return count; +} + +static const struct file_operations export_fops = { + .write = gator_pmu_export_write, +}; + +static int gator_pmu_create_files(struct super_block *sb, struct dentry *root, struct dentry *events) +{ + struct dentry *dir; + + gator_sb = sb; + gator_events_dir = events; + + gatorfs_create_file(sb, root, "pmu_init", &gator_pmu_init_fops); + + dir = gatorfs_mkdir(sb, root, "pmu"); + if (!dir) + return -1; + + gatorfs_create_file(sb, dir, "export", &export_fops); + + dir = gatorfs_mkdir(sb, root, "uncore_pmu"); + if (!dir) + return -1; + + gatorfs_create_file(sb, dir, "export", &export_fops); + + return 0; +} + +static void gator_pmu_exit(void) +{ + mutex_lock(&pmu_mutex); + { + struct gator_cpu *gator_cpu; + struct gator_cpu *next; + + list_for_each_entry_safe(gator_cpu, next, &gator_cpus, list) { + kfree(gator_cpu); + } + } + { + struct uncore_pmu *uncore_pmu; + struct uncore_pmu *next; + + list_for_each_entry_safe(uncore_pmu, next, &uncore_pmus, list) { + struct cpumask * cpumask = (struct cpumask *) GATOR_ATOMIC_READ(&(uncore_pmu->cpumask_atomic)); + if (cpumask) { + kfree(cpumask); + } + + kfree(uncore_pmu); + } + } + mutex_unlock(&pmu_mutex); +} diff --git a/drivers/gator/gator_trace_gpu.c b/drivers/gator/gator_trace_gpu.c new file mode 100644 index 000000000000..0b3424a852a9 --- /dev/null +++ b/drivers/gator/gator_trace_gpu.c @@ -0,0 +1,330 @@ +/** + * Copyright (C) Arm Limited 2010-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include "gator.h" + +#include +#include +#include +#include + +#ifdef MALI_SUPPORT +#ifdef MALI_DIR_MIDGARD +/* New DDK Directory structure with kernel/drivers/gpu/arm/midgard*/ +#include "mali_linux_trace.h" +#else +/* Old DDK Directory structure with kernel/drivers/gpu/arm/t6xx*/ +#include "linux/mali_linux_trace.h" +#endif +#endif + +/* + * Taken from MALI_PROFILING_EVENT_TYPE_* items in Mali DDK. + */ +#define EVENT_TYPE_SINGLE 0 +#define EVENT_TYPE_START 1 +#define EVENT_TYPE_STOP 2 +#define EVENT_TYPE_SUSPEND 3 +#define EVENT_TYPE_RESUME 4 + +/* Note whether tracepoints have been registered */ +static int mali_timeline_trace_registered; +static int mali_job_slots_trace_registered; + +enum { + GPU_UNIT_NONE = 0, + GPU_UNIT_VP, + GPU_UNIT_FP, + GPU_UNIT_CL, + NUMBER_OF_GPU_UNITS +}; + +#if defined(MALI_SUPPORT) + +struct mali_activity { + int core; + int key; + int count; + int last_activity; + int last_pid; +}; + +// Midgard can have up to 16 cores, but Bifrost can have up to 32, so reserving the max. +#define NUMBER_OF_GPU_CORES 32 + +static struct mali_activity mali_activities[NUMBER_OF_GPU_UNITS*NUMBER_OF_GPU_CORES]; +static DEFINE_SPINLOCK(mali_activities_lock); + +/* Only one event should be running on a unit and core at a time (ie, + * a start event can only be followed by a stop and vice versa), but + * because the kernel only knows when a job is enqueued and not + * started, it is possible for a start1, start2, stop1, stop2. Change + * it back into start1, stop1, start2, stop2 by queueing up start2 and + * releasing it when stop1 is received. + */ + +static int mali_activity_index(int core, int key) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(mali_activities); ++i) { + if ((mali_activities[i].core == core) && (mali_activities[i].key == key)) + break; + if ((mali_activities[i].core == 0) && (mali_activities[i].key == 0)) { + mali_activities[i].core = core; + mali_activities[i].key = key; + break; + } + } + BUG_ON(i >= ARRAY_SIZE(mali_activities)); + + return i; +} + +static void mali_activity_enqueue(int core, int key, int activity, int pid) +{ + int i; + int count; + + spin_lock(&mali_activities_lock); + i = mali_activity_index(core, key); + + count = mali_activities[i].count; + BUG_ON(count < 0 || count > 2); + if (count > 1) + /* + * The last value is about to be overwritten, send it now. This + * may happen if a stop message is lost. + */ + gator_marshal_activity_switch(core, key, mali_activities[i].last_activity, mali_activities[i].last_pid); + else + ++mali_activities[i].count; + if (count) { + mali_activities[i].last_activity = activity; + mali_activities[i].last_pid = pid; + } + spin_unlock(&mali_activities_lock); + + if (!count) + gator_marshal_activity_switch(core, key, activity, pid); +} + +static void mali_activity_stop(int core, int key) +{ + int i; + int count; + int last_activity = 0; + int last_pid = 0; + + spin_lock(&mali_activities_lock); + i = mali_activity_index(core, key); + + if (mali_activities[i].count == 0) { + spin_unlock(&mali_activities_lock); + return; + } + --mali_activities[i].count; + count = mali_activities[i].count; + if (count) { + last_activity = mali_activities[i].last_activity; + last_pid = mali_activities[i].last_pid; + } + spin_unlock(&mali_activities_lock); + + gator_marshal_activity_switch(core, key, 0, 0); + if (count) + gator_marshal_activity_switch(core, key, last_activity, last_pid); +} + +static void mali_activity_clear(struct mali_counter mali_activity[], size_t mali_activity_size) +{ + int activity; + int cores; + int core; + + for (activity = 0; activity < mali_activity_size; ++activity) { + cores = mali_activity[activity].cores; + if (cores < 0) + cores = 1; + for (core = 0; core < cores; ++core) { + if (mali_activity[activity].enabled) { + preempt_disable(); + gator_marshal_activity_switch(core, mali_activity[activity].key, 0, 0); + preempt_enable(); + } + } + } +} + +#endif + +#if defined(MALI_SUPPORT) && (MALI_SUPPORT != MALI_MIDGARD_OR_BIFROST) +#include "gator_events_mali_4xx.h" + +/* + * Taken from MALI_PROFILING_EVENT_CHANNEL_* in Mali DDK. + */ +enum { + EVENT_CHANNEL_SOFTWARE = 0, + EVENT_CHANNEL_VP0 = 1, + EVENT_CHANNEL_FP0 = 5, + EVENT_CHANNEL_FP1, + EVENT_CHANNEL_FP2, + EVENT_CHANNEL_FP3, + EVENT_CHANNEL_FP4, + EVENT_CHANNEL_FP5, + EVENT_CHANNEL_FP6, + EVENT_CHANNEL_FP7, + EVENT_CHANNEL_GPU = 21 +}; + +/** + * These events are applicable when the type MALI_PROFILING_EVENT_TYPE_SINGLE is used from the GPU channel + */ +enum { + EVENT_REASON_SINGLE_GPU_NONE = 0, + EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE = 1, +}; + +struct mali_counter mali_activity[2]; + +GATOR_DEFINE_PROBE(mali_timeline_event, TP_PROTO(unsigned int event_id, unsigned int d0, unsigned int d1, unsigned int d2, unsigned int d3, unsigned int d4)) +{ + unsigned int component, state; + + /* do as much work as possible before disabling interrupts */ + component = (event_id >> 16) & 0xFF; /* component is an 8-bit field */ + state = (event_id >> 24) & 0xF; /* state is a 4-bit field */ + + switch (state) { + case EVENT_TYPE_START: + if (component == EVENT_CHANNEL_VP0) { + /* tgid = d0; pid = d1; */ + if (mali_activity[1].enabled) + mali_activity_enqueue(0, mali_activity[1].key, 1, d1); + } else if (component >= EVENT_CHANNEL_FP0 && component <= EVENT_CHANNEL_FP7) { + /* tgid = d0; pid = d1; */ + if (mali_activity[0].enabled) + mali_activity_enqueue(component - EVENT_CHANNEL_FP0, mali_activity[0].key, 1, d1); + } + break; + + case EVENT_TYPE_STOP: + if (component == EVENT_CHANNEL_VP0) { + if (mali_activity[1].enabled) + mali_activity_stop(0, mali_activity[1].key); + } else if (component >= EVENT_CHANNEL_FP0 && component <= EVENT_CHANNEL_FP7) { + if (mali_activity[0].enabled) + mali_activity_stop(component - EVENT_CHANNEL_FP0, mali_activity[0].key); + } + break; + + case EVENT_TYPE_SINGLE: + if (component == EVENT_CHANNEL_GPU) { + unsigned int reason = (event_id & 0xffff); + + if (reason == EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE) + gator_events_mali_log_dvfs_event(d0, d1); + } + break; + + default: + break; + } +} +#endif + +#if defined(MALI_SUPPORT) && (MALI_SUPPORT == MALI_MIDGARD_OR_BIFROST) + +struct mali_counter mali_activity[3]; + +#if defined(MALI_JOB_SLOTS_EVENT_CHANGED) +GATOR_DEFINE_PROBE(mali_job_slots_event, TP_PROTO(unsigned int event_id, unsigned int tgid, unsigned int pid, unsigned char job_id)) +#else +GATOR_DEFINE_PROBE(mali_job_slots_event, TP_PROTO(unsigned int event_id, unsigned int tgid, unsigned int pid)) +#endif +{ + unsigned int component, state, unit; +#if !defined(MALI_JOB_SLOTS_EVENT_CHANGED) + unsigned char job_id = 0; +#endif + + component = (event_id >> 16) & 0xFF; /* component is an 8-bit field */ + state = (event_id >> 24) & 0xF; /* state is a 4-bit field */ + + switch (component) { + case 0: + unit = GPU_UNIT_FP; + break; + case 1: + unit = GPU_UNIT_VP; + break; + case 2: + unit = GPU_UNIT_CL; + break; + default: + unit = GPU_UNIT_NONE; + } + + if (unit != GPU_UNIT_NONE) { + switch (state) { + case EVENT_TYPE_START: + if (mali_activity[component].enabled) + mali_activity_enqueue(0, mali_activity[component].key, 1, (pid != 0 ? pid : tgid)); + break; + case EVENT_TYPE_STOP: + default: /* Some jobs can be soft-stopped, so ensure that this terminates the activity trace. */ + if (mali_activity[component].enabled) + mali_activity_stop(0, mali_activity[component].key); + break; + } + } +} +#endif + +static int gator_trace_gpu_start(void) +{ + /* + * Returns nonzero for installation failed + * Absence of gpu trace points is not an error + */ + +#if defined(MALI_SUPPORT) + memset(&mali_activities, 0, sizeof(mali_activities)); +#endif + mali_timeline_trace_registered = mali_job_slots_trace_registered = 0; + +#if defined(MALI_SUPPORT) && (MALI_SUPPORT != MALI_MIDGARD_OR_BIFROST) + mali_activity_clear(mali_activity, ARRAY_SIZE(mali_activity)); + if (!GATOR_REGISTER_TRACE(mali_timeline_event)) + mali_timeline_trace_registered = 1; +#endif + +#if defined(MALI_SUPPORT) && (MALI_SUPPORT == MALI_MIDGARD_OR_BIFROST) + mali_activity_clear(mali_activity, ARRAY_SIZE(mali_activity)); + if (!GATOR_REGISTER_TRACE(mali_job_slots_event)) + mali_job_slots_trace_registered = 1; +#endif + + return 0; +} + +static void gator_trace_gpu_stop(void) +{ +#if defined(MALI_SUPPORT) && (MALI_SUPPORT != MALI_MIDGARD_OR_BIFROST) + if (mali_timeline_trace_registered) + GATOR_UNREGISTER_TRACE(mali_timeline_event); +#endif + +#if defined(MALI_SUPPORT) && (MALI_SUPPORT == MALI_MIDGARD_OR_BIFROST) + if (mali_job_slots_trace_registered) + GATOR_UNREGISTER_TRACE(mali_job_slots_event); +#endif + + mali_timeline_trace_registered = mali_job_slots_trace_registered = 0; +} diff --git a/drivers/gator/gator_trace_power.c b/drivers/gator/gator_trace_power.c new file mode 100644 index 000000000000..39649668298d --- /dev/null +++ b/drivers/gator/gator_trace_power.c @@ -0,0 +1,195 @@ +/** + * Copyright (C) Arm Limited 2011-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#include +#include + +#if defined(__arm__) + +#include + +#define implements_wfi() (!machine_is_omap3_beagle()) + +#else + +#define implements_wfi() false + +#endif + +/* cpu_frequency and cpu_idle trace points were introduced in Linux + * kernel v2.6.38 the now deprecated power_frequency trace point was + * available prior to 2.6.38, but only for x86 + */ +#if GATOR_CPU_FREQ_SUPPORT + +static DEFINE_PER_CPU(ulong, idle_prev_state); +static ulong power_cpu_enabled[GATOR_CLUSTER_COUNT]; +static bool power_cpu_enabled_any; +static ulong power_cpu_key[GATOR_CLUSTER_COUNT]; + +static int gator_trace_power_create_files(struct super_block *sb, struct dentry *root) +{ + struct dentry *dir; + int cpu; + bool found_nonzero_freq = false; + + /* Even if CONFIG_CPU_FREQ is defined, it still may not be + * used. Check for non-zero values from cpufreq_quick_get + */ + for_each_online_cpu(cpu) { + if (cpufreq_quick_get(cpu) > 0) { + found_nonzero_freq = true; + break; + } + } + + if (found_nonzero_freq) { + char buf[40]; + int i; + + /* cpu_frequency */ + for (i = 0; i < gator_cluster_count; i++) { + snprintf(buf, sizeof(buf), "%s_freq", gator_clusters[i]->pmnc_name); + dir = gatorfs_mkdir(sb, root, buf); + if (!dir) + return -1; + gatorfs_create_ulong(sb, dir, "enabled", &power_cpu_enabled[i]); + gatorfs_create_ro_ulong(sb, dir, "key", &power_cpu_key[i]); + } + } + + return 0; +} + +/* 'cpu' may not equal smp_processor_id(), i.e. may not be running on the core that is having the freq/idle state change */ +GATOR_DEFINE_PROBE(cpu_frequency, TP_PROTO(unsigned int frequency, unsigned int cpu)) +{ + cpu = lcpu_to_pcpu(cpu); + marshal_event_single64(cpu, power_cpu_key[gator_clusterids[cpu]], frequency * 1000L); +} + +GATOR_DEFINE_PROBE(cpu_idle, TP_PROTO(unsigned int state, unsigned int cpu)) +{ + cpu = lcpu_to_pcpu(cpu); + + if (state == per_cpu(idle_prev_state, cpu)) + return; + + if (implements_wfi()) + marshal_idle(cpu, state); + + per_cpu(idle_prev_state, cpu) = state; +} + +static void gator_trace_power_online(void) +{ + int pcpu = get_physical_cpu(); + int lcpu = get_logical_cpu(); + + if (power_cpu_enabled[gator_clusterids[pcpu]]) + marshal_event_single64(pcpu, power_cpu_key[gator_clusterids[pcpu]], cpufreq_quick_get(lcpu) * 1000L); +} + +static void gator_trace_power_offline(void) +{ + /* Set frequency to zero on an offline */ + int cpu = get_physical_cpu(); + + if (power_cpu_enabled[gator_clusterids[cpu]]) + marshal_event_single(cpu, power_cpu_key[gator_clusterids[cpu]], 0); +} + +static int gator_trace_power_start(void) +{ + int cpu; + int i; + + power_cpu_enabled_any = false; + for (i = 0; i < gator_cluster_count; i++) { + if (power_cpu_enabled[i]) { + power_cpu_enabled_any = true; + break; + } + } + + /* register tracepoints */ + if (power_cpu_enabled_any) + if (GATOR_REGISTER_TRACE(cpu_frequency)) + goto fail_cpu_frequency_exit; + + /* Always register for cpu_idle for detecting WFI */ + if (GATOR_REGISTER_TRACE(cpu_idle)) + goto fail_cpu_idle_exit; + pr_debug("gator: registered power event tracepoints\n"); + + for_each_present_cpu(cpu) { + per_cpu(idle_prev_state, cpu) = 0; + } + + return 0; + + /* unregister tracepoints on error */ +fail_cpu_idle_exit: + if (power_cpu_enabled_any) + GATOR_UNREGISTER_TRACE(cpu_frequency); +fail_cpu_frequency_exit: + pr_err("gator: power event tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n"); + + return -1; +} + +static void gator_trace_power_stop(void) +{ + int i; + + if (power_cpu_enabled_any) + GATOR_UNREGISTER_TRACE(cpu_frequency); + GATOR_UNREGISTER_TRACE(cpu_idle); + pr_debug("gator: unregistered power event tracepoints\n"); + + for (i = 0; i < gator_cluster_count; i++) + power_cpu_enabled[i] = 0; +} + +static void gator_trace_power_init(void) +{ + int i; + + for (i = 0; i < gator_cluster_count; i++) { + power_cpu_enabled[i] = 0; + power_cpu_key[i] = gator_events_get_key(); + } +} +#else +static int gator_trace_power_create_files(struct super_block *sb, struct dentry *root) +{ + return 0; +} + +static void gator_trace_power_online(void) +{ +} + +static void gator_trace_power_offline(void) +{ +} + +static int gator_trace_power_start(void) +{ + return 0; +} + +static void gator_trace_power_stop(void) +{ +} + +static void gator_trace_power_init(void) +{ +} +#endif diff --git a/drivers/gator/gator_trace_sched.c b/drivers/gator/gator_trace_sched.c new file mode 100644 index 000000000000..3c6b2837f465 --- /dev/null +++ b/drivers/gator/gator_trace_sched.c @@ -0,0 +1,341 @@ +/** + * Copyright (C) Arm Limited 2010-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#include +#include + +#include "gator.h" + +#define TASK_MAP_ENTRIES 1024 /* must be power of 2 */ +#define TASK_MAX_COLLISIONS 2 + +enum { + STATE_WAIT_ON_OTHER = 0, + STATE_CONTENTION, + STATE_WAIT_ON_IO, + CPU_WAIT_TOTAL +}; + +static DEFINE_PER_CPU(uint64_t *, taskname_keys); +static DEFINE_PER_CPU(int, collecting); + +/* this array is never read as the cpu charts are derived + * counters the files are needed, nonetheless, to show that these + * counters are available + */ +static const char *sched_wait_event_names[] = { + "Linux_cpu_wait_contention", + "Linux_cpu_wait_io", +}; +static ulong sched_wait_enabled[ARRAY_SIZE(sched_wait_event_names)]; +static ulong sched_wait_keys[ARRAY_SIZE(sched_wait_event_names)]; + +static const char *sched_activity_event_names[] = { + "system", + "user", +}; +static ulong sched_activity_enabled[ARRAY_SIZE(sched_activity_event_names)][GATOR_CLUSTER_COUNT]; +static ulong sched_activity_keys[ARRAY_SIZE(sched_activity_event_names)][GATOR_CLUSTER_COUNT]; + +static int sched_trace_create_files(struct super_block *sb, struct dentry *root) +{ + struct dentry *dir; + int i; + int j; + char buf[40]; + + for (i = 0; i < ARRAY_SIZE(sched_wait_event_names); ++i) { + dir = gatorfs_mkdir(sb, root, sched_wait_event_names[i]); + if (!dir) + return -1; + gatorfs_create_ulong(sb, dir, "enabled", &sched_wait_enabled[i]); + gatorfs_create_ro_ulong(sb, dir, "key", &sched_wait_keys[i]); + } + + for (i = 0; i < ARRAY_SIZE(sched_activity_event_names); ++i) { + for (j = 0; j < gator_cluster_count; j++) { + snprintf(buf, sizeof(buf), "%s_%s", gator_clusters[j]->pmnc_name, sched_activity_event_names[i]); + dir = gatorfs_mkdir(sb, root, buf); + if (!dir) + return -1; + gatorfs_create_ulong(sb, dir, "enabled", &sched_activity_enabled[i][j]); + gatorfs_create_ro_ulong(sb, dir, "key", &sched_activity_keys[i][j]); + } + } + + return 0; +} + +static void emit_pid_name(const char *comm, struct task_struct *task) +{ + bool found = false; + char taskcomm[TASK_COMM_LEN + 3]; + unsigned long x, cpu = get_physical_cpu(); + uint64_t *keys = &(per_cpu(taskname_keys, cpu)[(task->pid & 0xFF) * TASK_MAX_COLLISIONS]); + uint64_t value; + + value = gator_chksum_crc32(comm); + value = (value << 32) | (uint32_t)task->pid; + + /* determine if the thread name was emitted already */ + for (x = 0; x < TASK_MAX_COLLISIONS; x++) { + if (keys[x] == value) { + found = true; + break; + } + } + + if (!found) { + /* shift values, new value always in front */ + uint64_t oldv, newv = value; + + for (x = 0; x < TASK_MAX_COLLISIONS; x++) { + oldv = keys[x]; + keys[x] = newv; + newv = oldv; + } + + /* emit pid names, cannot use get_task_comm, as it's not exported on all kernel versions */ + if (strlcpy(taskcomm, comm, TASK_COMM_LEN) == TASK_COMM_LEN - 1) { + /* append ellipses if comm has length of TASK_COMM_LEN - 1 */ + strcat(taskcomm, "..."); + } + + marshal_thread_name(task->pid, taskcomm); + } +} + +static void collect_counters(u64 time, struct task_struct *task, bool sched_switch) +{ + int *buffer, len, cpu = get_physical_cpu(); + long long *buffer64; + struct gator_interface *gi; + + if (marshal_event_header(time)) { + list_for_each_entry(gi, &gator_events, list) { + if (gi->read) { + len = gi->read(&buffer, sched_switch); + if (len < 0) + pr_err("gator: read failed for %s\n", gi->name); + marshal_event(len, buffer); + } else if (gi->read64) { + len = gi->read64(&buffer64, sched_switch); + if (len < 0) + pr_err("gator: read64 failed for %s\n", gi->name); + marshal_event64(len, buffer64); + } + if (gi->read_proc && task != NULL) { + len = gi->read_proc(&buffer64, task); + if (len < 0) + pr_err("gator: read_proc failed for %s\n", gi->name); + marshal_event64(len, buffer64); + } + } + if (cpu == 0) + gator_emit_perf_time(time); + /* Only check after writing all counters so that time and corresponding counters appear in the same frame */ + buffer_check(cpu, BLOCK_COUNTER_BUF, time); + + /* Commit buffers on timeout */ + if (gator_live_rate > 0 && time >= per_cpu(gator_buffer_commit_time, cpu)) { + static const int buftypes[] = { NAME_BUF, COUNTER_BUF, BLOCK_COUNTER_BUF, SCHED_TRACE_BUF, ACTIVITY_BUF }; + int i; + + for (i = 0; i < ARRAY_SIZE(buftypes); ++i) + gator_commit_buffer(cpu, buftypes[i], time); + + /* spinlocks are noops on uniprocessor machines and mutexes do + * not work in sched_switch context in RT-Preempt full, so + * disable proactive flushing of the annotate frame on + * uniprocessor machines. + */ +#ifdef CONFIG_SMP + /* Try to preemptively flush the annotate buffer to reduce the chance of the buffer being full */ + if (on_primary_core() && spin_trylock(&annotate_lock)) { + gator_commit_buffer(0, ANNOTATE_BUF, time); + spin_unlock(&annotate_lock); + } +#endif + } + } +} + +/* special case used during a suspend of the system */ +static void trace_sched_insert_idle(void) +{ + marshal_sched_trace_switch(0, 0); +} + +static void gator_trace_emit_link(struct task_struct *p) +{ + int cookie; + int cpu = get_physical_cpu(); + + cookie = get_exec_cookie(cpu, p); + emit_pid_name(p->comm, p); + + marshal_link(cookie, p->tgid, p->pid); +} + +GATOR_DEFINE_PROBE(sched_process_fork, TP_PROTO(struct task_struct *parent, struct task_struct *child)) +{ + gator_trace_emit_link(child); +} + +GATOR_DEFINE_PROBE(sched_process_exec, TP_PROTO(struct task_struct *p, pid_t old_pid, struct linux_binprm *bprm)) +{ + gator_trace_emit_link(p); +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0) +GATOR_DEFINE_PROBE(task_rename, TP_PROTO(struct task_struct *task, char *comm)) +#else +GATOR_DEFINE_PROBE(task_rename, TP_PROTO(struct task_struct *task, const char *comm)) +#endif +{ + emit_pid_name(comm, task); +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) +GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct task_struct *prev, struct task_struct *next)) +#else +GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(bool preempt, struct task_struct *prev, struct task_struct *next)) +#endif +{ + int state; + int cpu = get_physical_cpu(); + + per_cpu(in_scheduler_context, cpu) = true; + + /* do as much work as possible before disabling interrupts */ + if (prev->state == TASK_RUNNING) + state = STATE_CONTENTION; + else if (prev->in_iowait) + state = STATE_WAIT_ON_IO; + else + state = STATE_WAIT_ON_OTHER; + + per_cpu(collecting, cpu) = 1; + collect_counters(gator_get_time(), prev, true); + per_cpu(collecting, cpu) = 0; + + marshal_sched_trace_switch(next->pid, state); + + per_cpu(in_scheduler_context, cpu) = false; +} + +GATOR_DEFINE_PROBE(sched_process_free, TP_PROTO(struct task_struct *p)) +{ + marshal_sched_trace_exit(p->tgid, p->pid); +} + +static void do_nothing(void *info) +{ + /* Intentionally do nothing */ + (void)info; +} + +static int register_scheduler_tracepoints(void) +{ + /* register tracepoints */ + if (GATOR_REGISTER_TRACE(sched_process_fork)) + goto fail_sched_process_fork; + if (GATOR_REGISTER_TRACE(sched_process_exec)) + goto fail_sched_process_exec; + if (GATOR_REGISTER_TRACE(task_rename)) + goto fail_task_rename; + if (GATOR_REGISTER_TRACE(sched_switch)) + goto fail_sched_switch; + if (GATOR_REGISTER_TRACE(sched_process_free)) + goto fail_sched_process_free; + pr_debug("gator: registered tracepoints\n"); + + /* Now that the scheduler tracepoint is registered, force a context + * switch on all cpus to capture what is currently running. + */ + on_each_cpu(do_nothing, NULL, 0); + + return 0; + + /* unregister tracepoints on error */ +fail_sched_process_free: + GATOR_UNREGISTER_TRACE(sched_switch); +fail_sched_switch: + GATOR_UNREGISTER_TRACE(task_rename); +fail_task_rename: + GATOR_UNREGISTER_TRACE(sched_process_exec); +fail_sched_process_exec: + GATOR_UNREGISTER_TRACE(sched_process_fork); +fail_sched_process_fork: + pr_err("gator: tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n"); + + return -1; +} + +static void unregister_scheduler_tracepoints(void) +{ + GATOR_UNREGISTER_TRACE(sched_process_fork); + GATOR_UNREGISTER_TRACE(sched_process_exec); + GATOR_UNREGISTER_TRACE(task_rename); + GATOR_UNREGISTER_TRACE(sched_switch); + GATOR_UNREGISTER_TRACE(sched_process_free); + pr_debug("gator: unregistered tracepoints\n"); +} + +static void gator_trace_sched_stop(void) +{ + int cpu; + + unregister_scheduler_tracepoints(); + + for_each_present_cpu(cpu) { + kfree(per_cpu(taskname_keys, cpu)); + } +} + +static int gator_trace_sched_start(void) +{ + int cpu, size; + int ret; + + for_each_present_cpu(cpu) { + size = TASK_MAP_ENTRIES * TASK_MAX_COLLISIONS * sizeof(uint64_t); + per_cpu(taskname_keys, cpu) = kmalloc(size, GFP_KERNEL); + if (!per_cpu(taskname_keys, cpu)) + return -1; + memset(per_cpu(taskname_keys, cpu), 0, size); + } + + ret = register_scheduler_tracepoints(); + + return ret; +} + +static void gator_trace_sched_offline(void) +{ + trace_sched_insert_idle(); +} + +static void gator_trace_sched_init(void) +{ + int i; + int j; + + for (i = 0; i < ARRAY_SIZE(sched_wait_enabled); i++) { + sched_wait_enabled[i] = 0; + sched_wait_keys[i] = gator_events_get_key(); + } + + for (i = 0; i < ARRAY_SIZE(sched_activity_enabled); i++) { + for (j = 0; j < gator_cluster_count; j++) { + sched_activity_enabled[i][j] = 0; + sched_activity_keys[i][j] = gator_events_get_key(); + } + } +} diff --git a/drivers/gator/mali/mali_mjollnir_profiling_gator_api.h b/drivers/gator/mali/mali_mjollnir_profiling_gator_api.h new file mode 100644 index 000000000000..c760bef81dd3 --- /dev/null +++ b/drivers/gator/mali/mali_mjollnir_profiling_gator_api.h @@ -0,0 +1,159 @@ +/** + * Copyright (C) Arm Limited 2013-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#ifndef __MALI_MJOLLNIR_PROFILING_GATOR_API_H__ +#define __MALI_MJOLLNIR_PROFILING_GATOR_API_H__ + +#ifdef __cplusplus +extern "C" +{ +#endif + + +/* + * The number of processor cores. Update to suit your hardware implementation. + */ +#define MAX_NUM_FP_CORES (4) +#define MAX_NUM_VP_CORES (1) +#define MAX_NUM_L2_CACHE_CORES (1) + +enum counters { + /* Timeline activity */ + ACTIVITY_VP_0 = 0, + ACTIVITY_FP_0, + ACTIVITY_FP_1, + ACTIVITY_FP_2, + ACTIVITY_FP_3, + + /* L2 cache counters */ + COUNTER_L2_0_C0, + COUNTER_L2_0_C1, + + /* Vertex processor counters */ + COUNTER_VP_0_C0, + COUNTER_VP_0_C1, + + /* Fragment processor counters */ + COUNTER_FP_0_C0, + COUNTER_FP_0_C1, + COUNTER_FP_1_C0, + COUNTER_FP_1_C1, + COUNTER_FP_2_C0, + COUNTER_FP_2_C1, + COUNTER_FP_3_C0, + COUNTER_FP_3_C1, + + /* EGL Software Counters */ + COUNTER_EGL_BLIT_TIME, + + /* GLES Software Counters */ + COUNTER_GLES_DRAW_ELEMENTS_CALLS, + COUNTER_GLES_DRAW_ELEMENTS_NUM_INDICES, + COUNTER_GLES_DRAW_ELEMENTS_NUM_TRANSFORMED, + COUNTER_GLES_DRAW_ARRAYS_CALLS, + COUNTER_GLES_DRAW_ARRAYS_NUM_TRANSFORMED, + COUNTER_GLES_DRAW_POINTS, + COUNTER_GLES_DRAW_LINES, + COUNTER_GLES_DRAW_LINE_LOOP, + COUNTER_GLES_DRAW_LINE_STRIP, + COUNTER_GLES_DRAW_TRIANGLES, + COUNTER_GLES_DRAW_TRIANGLE_STRIP, + COUNTER_GLES_DRAW_TRIANGLE_FAN, + COUNTER_GLES_NON_VBO_DATA_COPY_TIME, + COUNTER_GLES_UNIFORM_BYTES_COPIED_TO_MALI, + COUNTER_GLES_UPLOAD_TEXTURE_TIME, + COUNTER_GLES_UPLOAD_VBO_TIME, + COUNTER_GLES_NUM_FLUSHES, + COUNTER_GLES_NUM_VSHADERS_GENERATED, + COUNTER_GLES_NUM_FSHADERS_GENERATED, + COUNTER_GLES_VSHADER_GEN_TIME, + COUNTER_GLES_FSHADER_GEN_TIME, + COUNTER_GLES_INPUT_TRIANGLES, + COUNTER_GLES_VXCACHE_HIT, + COUNTER_GLES_VXCACHE_MISS, + COUNTER_GLES_VXCACHE_COLLISION, + COUNTER_GLES_CULLED_TRIANGLES, + COUNTER_GLES_CULLED_LINES, + COUNTER_GLES_BACKFACE_TRIANGLES, + COUNTER_GLES_GBCLIP_TRIANGLES, + COUNTER_GLES_GBCLIP_LINES, + COUNTER_GLES_TRIANGLES_DRAWN, + COUNTER_GLES_DRAWCALL_TIME, + COUNTER_GLES_TRIANGLES_COUNT, + COUNTER_GLES_INDEPENDENT_TRIANGLES_COUNT, + COUNTER_GLES_STRIP_TRIANGLES_COUNT, + COUNTER_GLES_FAN_TRIANGLES_COUNT, + COUNTER_GLES_LINES_COUNT, + COUNTER_GLES_INDEPENDENT_LINES_COUNT, + COUNTER_GLES_STRIP_LINES_COUNT, + COUNTER_GLES_LOOP_LINES_COUNT, + + COUNTER_FILMSTRIP, + COUNTER_FREQUENCY, + COUNTER_VOLTAGE, + + NUMBER_OF_EVENTS +}; + +#define FIRST_ACTIVITY_EVENT ACTIVITY_VP_0 +#define LAST_ACTIVITY_EVENT ACTIVITY_FP_3 + +#define FIRST_HW_COUNTER COUNTER_L2_0_C0 +#define LAST_HW_COUNTER COUNTER_FP_3_C1 + +#define FIRST_SW_COUNTER COUNTER_EGL_BLIT_TIME +#define LAST_SW_COUNTER COUNTER_GLES_LOOP_LINES_COUNT + +/* Signifies that the system is able to report voltage and frequency numbers. */ +#define DVFS_REPORTED_BY_DDK 1 + +/** + * Structure to pass performance counter data of a Mali core + */ +struct _mali_profiling_core_counters { + u32 source0; + u32 value0; + u32 source1; + u32 value1; +}; + +/* + * For compatibility with utgard. + */ +struct _mali_profiling_l2_counter_values { + struct _mali_profiling_core_counters cores[MAX_NUM_L2_CACHE_CORES]; +}; + +struct _mali_profiling_mali_version { + u32 mali_product_id; + u32 mali_version_major; + u32 mali_version_minor; + u32 num_of_l2_cores; + u32 num_of_fp_cores; + u32 num_of_vp_cores; +}; + +extern void _mali_profiling_get_mali_version(struct _mali_profiling_mali_version *values); +extern u32 _mali_profiling_get_l2_counters(struct _mali_profiling_l2_counter_values *values); + +/* + * List of possible actions allowing DDK to be controlled by Streamline. + * The following numbers are used by DDK to control the frame buffer dumping. + */ +#define FBDUMP_CONTROL_ENABLE (1) +#define FBDUMP_CONTROL_RATE (2) +#define SW_COUNTER_ENABLE (3) +#define FBDUMP_CONTROL_RESIZE_FACTOR (4) + + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_MJOLLNIR_PROFILING_GATOR_API_H__ */ diff --git a/drivers/gator/mali/mali_utgard_profiling_gator_api.h b/drivers/gator/mali/mali_utgard_profiling_gator_api.h new file mode 100644 index 000000000000..aedc1da35d5f --- /dev/null +++ b/drivers/gator/mali/mali_utgard_profiling_gator_api.h @@ -0,0 +1,197 @@ +/** + * Copyright (C) Arm Limited 2013-2016. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#ifndef __MALI_UTGARD_PROFILING_GATOR_API_H__ +#define __MALI_UTGARD_PROFILING_GATOR_API_H__ + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define MALI_PROFILING_API_VERSION 4 + +#define MAX_NUM_L2_CACHE_CORES 3 +#define MAX_NUM_FP_CORES 8 +#define MAX_NUM_VP_CORES 1 + +/** The list of events supported by the Mali DDK. */ +enum { + /* Vertex processor activity */ + ACTIVITY_VP_0 = 0, + + /* Fragment processor activity */ + ACTIVITY_FP_0, /* 1 */ + ACTIVITY_FP_1, + ACTIVITY_FP_2, + ACTIVITY_FP_3, + ACTIVITY_FP_4, + ACTIVITY_FP_5, + ACTIVITY_FP_6, + ACTIVITY_FP_7, + + /* L2 cache counters */ + COUNTER_L2_0_C0, + COUNTER_L2_0_C1, + COUNTER_L2_1_C0, + COUNTER_L2_1_C1, + COUNTER_L2_2_C0, + COUNTER_L2_2_C1, + + /* Vertex processor counters */ + COUNTER_VP_0_C0, /*15*/ + COUNTER_VP_0_C1, + + /* Fragment processor counters */ + COUNTER_FP_0_C0, + COUNTER_FP_0_C1, + COUNTER_FP_1_C0, + COUNTER_FP_1_C1, + COUNTER_FP_2_C0, + COUNTER_FP_2_C1, + COUNTER_FP_3_C0, + COUNTER_FP_3_C1, + COUNTER_FP_4_C0, + COUNTER_FP_4_C1, + COUNTER_FP_5_C0, + COUNTER_FP_5_C1, + COUNTER_FP_6_C0, + COUNTER_FP_6_C1, + COUNTER_FP_7_C0, + COUNTER_FP_7_C1, /* 32 */ + + /* + * If more hardware counters are added, the _mali_osk_hw_counter_table + * below should also be updated. + */ + + /* EGL software counters */ + COUNTER_EGL_BLIT_TIME, + + /* GLES software counters */ + COUNTER_GLES_DRAW_ELEMENTS_CALLS, + COUNTER_GLES_DRAW_ELEMENTS_NUM_INDICES, + COUNTER_GLES_DRAW_ELEMENTS_NUM_TRANSFORMED, + COUNTER_GLES_DRAW_ARRAYS_CALLS, + COUNTER_GLES_DRAW_ARRAYS_NUM_TRANSFORMED, + COUNTER_GLES_DRAW_POINTS, + COUNTER_GLES_DRAW_LINES, + COUNTER_GLES_DRAW_LINE_LOOP, + COUNTER_GLES_DRAW_LINE_STRIP, + COUNTER_GLES_DRAW_TRIANGLES, + COUNTER_GLES_DRAW_TRIANGLE_STRIP, + COUNTER_GLES_DRAW_TRIANGLE_FAN, + COUNTER_GLES_NON_VBO_DATA_COPY_TIME, + COUNTER_GLES_UNIFORM_BYTES_COPIED_TO_MALI, + COUNTER_GLES_UPLOAD_TEXTURE_TIME, + COUNTER_GLES_UPLOAD_VBO_TIME, + COUNTER_GLES_NUM_FLUSHES, + COUNTER_GLES_NUM_VSHADERS_GENERATED, + COUNTER_GLES_NUM_FSHADERS_GENERATED, + COUNTER_GLES_VSHADER_GEN_TIME, + COUNTER_GLES_FSHADER_GEN_TIME, + COUNTER_GLES_INPUT_TRIANGLES, + COUNTER_GLES_VXCACHE_HIT, + COUNTER_GLES_VXCACHE_MISS, + COUNTER_GLES_VXCACHE_COLLISION, + COUNTER_GLES_CULLED_TRIANGLES, + COUNTER_GLES_CULLED_LINES, + COUNTER_GLES_BACKFACE_TRIANGLES, + COUNTER_GLES_GBCLIP_TRIANGLES, + COUNTER_GLES_GBCLIP_LINES, + COUNTER_GLES_TRIANGLES_DRAWN, + COUNTER_GLES_DRAWCALL_TIME, + COUNTER_GLES_TRIANGLES_COUNT, + COUNTER_GLES_INDEPENDENT_TRIANGLES_COUNT, + COUNTER_GLES_STRIP_TRIANGLES_COUNT, + COUNTER_GLES_FAN_TRIANGLES_COUNT, + COUNTER_GLES_LINES_COUNT, + COUNTER_GLES_INDEPENDENT_LINES_COUNT, + COUNTER_GLES_STRIP_LINES_COUNT, + COUNTER_GLES_LOOP_LINES_COUNT, + + /* Framebuffer capture pseudo-counter */ + COUNTER_FILMSTRIP, + + NUMBER_OF_EVENTS +} _mali_osk_counter_id; + +#define FIRST_ACTIVITY_EVENT ACTIVITY_VP_0 +#define LAST_ACTIVITY_EVENT ACTIVITY_FP_7 + +#define FIRST_HW_COUNTER COUNTER_L2_0_C0 +#define LAST_HW_COUNTER COUNTER_FP_7_C1 + +#define FIRST_SW_COUNTER COUNTER_EGL_BLIT_TIME +#define LAST_SW_COUNTER COUNTER_GLES_LOOP_LINES_COUNT + +#define FIRST_SPECIAL_COUNTER COUNTER_FILMSTRIP +#define LAST_SPECIAL_COUNTER COUNTER_FILMSTRIP + +/** + * Structure to pass performance counter data of a Mali core + */ +struct _mali_profiling_core_counters { + u32 source0; + u32 value0; + u32 source1; + u32 value1; +}; + +/** + * Structure to pass performance counter data of Mali L2 cache cores + */ +struct _mali_profiling_l2_counter_values { + struct _mali_profiling_core_counters cores[MAX_NUM_L2_CACHE_CORES]; +}; + +/** + * Structure to pass data defining Mali instance in use: + * + * mali_product_id - Mali product id + * mali_version_major - Mali version major number + * mali_version_minor - Mali version minor number + * num_of_l2_cores - number of L2 cache cores + * num_of_fp_cores - number of fragment processor cores + * num_of_vp_cores - number of vertex processor cores + */ +struct _mali_profiling_mali_version { + u32 mali_product_id; + u32 mali_version_major; + u32 mali_version_minor; + u32 num_of_l2_cores; + u32 num_of_fp_cores; + u32 num_of_vp_cores; +}; + +/* + * List of possible actions to be controlled by Streamline. + * The following numbers are used by gator to control the frame buffer dumping and s/w counter reporting. + * We cannot use the enums in mali_uk_types.h because they are unknown inside gator. + */ +#define FBDUMP_CONTROL_ENABLE (1) +#define FBDUMP_CONTROL_RATE (2) +#define SW_COUNTER_ENABLE (3) +#define FBDUMP_CONTROL_RESIZE_FACTOR (4) + +void _mali_profiling_control(u32 action, u32 value); + +u32 _mali_profiling_get_l2_counters(struct _mali_profiling_l2_counter_values *values); + +int _mali_profiling_set_event(u32 counter_id, s32 event_id); + +u32 _mali_profiling_get_api_version(void); + +void _mali_profiling_get_mali_version(struct _mali_profiling_mali_version *values); + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_UTGARD_PROFILING_GATOR_API_H__ */ diff --git a/drivers/gator/mali_midgard.mk b/drivers/gator/mali_midgard.mk new file mode 100644 index 000000000000..b1405e682f7c --- /dev/null +++ b/drivers/gator/mali_midgard.mk @@ -0,0 +1,49 @@ +# Defines for Mali-Midgard/Bifrost driver +EXTRA_CFLAGS += -DMALI_USE_UMP=1 \ + -DMALI_LICENSE_IS_GPL=1 \ + -DMALI_BASE_TRACK_MEMLEAK=0 \ + -DMALI_DEBUG=0 \ + -DMALI_ERROR_INJECT_ON=0 \ + -DMALI_CUSTOMER_RELEASE=1 \ + -DMALI_UNIT_TEST=0 \ + -DMALI_BACKEND_KERNEL=1 \ + -DMALI_NO_MALI=0 + +DDK_DIR ?= . + +GATOR_MALI_MIDGARD_PATH := $(shell echo $(CONFIG_GATOR_MALI_MIDGARD_PATH)) + +ifneq ($(wildcard $(DDK_DIR)/$(GATOR_MALI_MIDGARD_PATH)/mali_kbase_gator_api.h),) + # r5p0/Fluorine - ... + EXTRA_CFLAGS += -DMALI_SIMPLE_API=1 \ + -DMALI_DIR_MIDGARD=1 \ + -I$(DDK_DIR)/$(GATOR_MALI_MIDGARD_PATH) \ + +else + ifneq ($(wildcard $(DDK_DIR)/$(GATOR_MALI_MIDGARD_PATH)/kbase),) + # ? - r3p0 + KBASE_DIR = $(DDK_DIR)/$(GATOR_MALI_MIDGARD_PATH)/kbase + OSK_DIR = $(DDK_DIR)/$(GATOR_MALI_MIDGARD_PATH)/kbase/osk + else + ifneq ($(wildcard $(DDK_DIR)/$(GATOR_MALI_MIDGARD_PATH)),) + # r4p0/Europium - r4p1/Europium-Inc + KBASE_DIR = $(DDK_DIR)/$(GATOR_MALI_MIDGARD_PATH) + OSK_DIR = $(DDK_DIR)/$(GATOR_MALI_MIDGARD_PATH)/osk + EXTRA_CFLAGS += -DMALI_DIR_MIDGARD=1 + endif + endif + + UMP_DIR = $(DDK_DIR)/include/linux + + # Include directories in the DDK + EXTRA_CFLAGS += -I$(KBASE_DIR)/ \ + -I$(KBASE_DIR)/.. \ + -I$(OSK_DIR)/.. \ + -I$(UMP_DIR)/.. \ + -I$(DDK_DIR)/include \ + -I$(KBASE_DIR)/osk/src/linux/include \ + -I$(KBASE_DIR)/platform_dummy \ + -I$(KBASE_DIR)/src \ + -Idrivers/staging/android \ + +endif