0-Linux coredump捕获分析

1.制作debug-root

commit a4a3b5ba5a7bdb3fb98028793d62f37788c0790a
config: CONFIG_DEBUG use debug rootfs, sstrip real rootfs


9ad4d66b7aa27f7ee42281466662d51044001278
 debug: add debug-root for debug coredump and panicVIM 

系统编译加上-g3,在strip之前,将这些文文件存放到debug-root下,strip之后的为正常文文件系统。

vim 14.07/config/Config-build.in

    config DEBUG
        bool
        prompt "Compile packages with debugging info"
        default n
        help
          Adds -g3 to the CFLAGS

    config USE_SSTRIP
        bool "sstrip"
        depends on !USE_GLIBC
        depends on !USE_EGLIBC
        help
          This will install binaries stripped using sstrip

在strip之前保存debug固件

vim 14.07/include/package-ipkg.mk

    # save to debug root, before strip
    @mkdir -p $(TARGET_DEBUG_DIR)
    $(CP) $$(IDIR_$(1))/* $(TARGET_DEBUG_DIR)

TARGET_DEBUG_DIR使用的几个地方：

linye@linye-ubuntu:~/ziroom/ZMAX/ZHA0107/zrouter/14.07$ grep -rn TARGET_DEBUG_DIR ./
./rules.mk:109:TARGET_DEBUG_DIR:=$(TARGET_ROOTFS_DIR)/root-debug-$(BOARD)
./include/kernel-defaults.mk:141:   $(CP) $(LINUX_DIR)/vmlinux $(TARGET_DEBUG_DIR)/vmlinux.debug
./include/package-ipkg.mk:155:  @mkdir -p $(TARGET_DEBUG_DIR)
./include/package-ipkg.mk:156:  $(CP) $$(IDIR_$(1))/* $(TARGET_DEBUG_DIR)
匹配到二进制文件 ./include/.package-ipkg.mk.swp
./include/image.mk:251: (cd $(TARGET_DEBUG_DIR);tar -czf $(1) *)

root-mtk和root-debug-mtk

linye@linye-ubuntu:~/ziroom/ZMAX/ZHA0107/zrouter/14.07$ ls build_dir/target-mipsel_1004kc_uClibc-0.9.33.2/root-*
build_dir/target-mipsel_1004kc_uClibc-0.9.33.2/root-debug-mtk:
bin      dev  init  mnt      proc  root  sys  usr  vmlinux.debug  zgateway
CONTROL  etc  lib   overlay  rom   sbin  tmp  var  www

build_dir/target-mipsel_1004kc_uClibc-0.9.33.2/root-mtk:
bin  dev  etc  lib  mnt  overlay  proc  rom  root  sbin  sys  tmp  usr  var  www  zgateway

2.添加core dump选项

在/etc/profile里面添加ulimit -c unlimited

root@zihome:/tmp# cat /etc/profile 
#!/bin/sh
[ -f /etc/banner ] && cat /etc/banner

export PATH=/usr/bin:/usr/sbin:/bin:/sbin
export HOME=$(grep -e "^${USER:-root}:" /etc/passwd | cut -d ":" -f 6)
export HOME=${HOME:-/root}
export PS1='\u@\h:\w\$ '

# generate core file
ulimit -c unlimited

[ -x /bin/more ] || alias more=less
[ -x /usr/bin/vim ] && alias vi=vim || alias vim=vi

[ -z "$KSH_VERSION" -o \! -s /etc/mkshrc ] || . /etc/mkshrc

[ -x /usr/bin/arp ] || arp() { cat /proc/net/arp; }
[ -x /usr/bin/ldd ] || ldd() { LD_TRACE_LOADED_OBJECTS=1 $*; }

系统中我们设置了了kernel.core_pattern=/tmp/coredump/%e.%t.%p.%s.core,生生成的coredump文文件将存放在/tmp/coredump目目录下

可以查看

cat ./proc/sys/kernel/core_pattern

如下修改：vim ./package/base-files/files/etc/sysctl.conf

kernel.panic=3
kernel.panic_on_oops=1
kernel.core_pattern=/tmp/coredump/%e.%t.%p.%s.core

net.ipv4.conf.default.arp_ignore=1
net.ipv4.conf.all.arp_ignore=1
net.ipv4.ip_forward=1
net.ipv4.icmp_echo_ignore_broadcasts=1
net.ipv4.icmp_ignore_bogus_error_responses=1
net.ipv4.igmp_max_memberships=100
net.ipv4.tcp_ecn=0
net.ipv4.tcp_fin_timeout=30
net.ipv4.tcp_keepalive_time=120
net.ipv4.tcp_syncookies=1
net.ipv4.tcp_timestamps=1
net.ipv4.tcp_sack=1
net.ipv4.tcp_dsack=1
net.ipv4.tcp_mtu_probing=1

net.ipv6.conf.default.forwarding=1
net.ipv6.conf.all.forwarding=1

在内核设置好coredump的生产目录后，在zdetect模块里面会定时去查询/tmp/coredump/的目录变化，如果有coredump文件，就会发生ubus消息出来。

3.procd 的coredump参数

我们在/etc/init.d/的启动脚本里面总是会设置以下参数，是什么意思呢

#!/bin/sh /etc/rc.common

START=99
USE_PROCD=1

start_service() {
        local auto_enabled=$(uci -q get zqos.wan.auto_enabled)
        local web_enabled=$(uci -q get zqos.wan.web_enabled)
        if [ $auto_enabled == 1 -a $web_enabled == 0 ]; then
                uci set zqos.wan.enabled=0
                uci set zqos.wan.auto_enabled=0
                uci commit zqos
                /etc/init.d/zqos restart
        fi

        procd_open_instance
        procd_set_param command /usr/bin/zspeedtest
        procd_set_param respawn
        [ -e /proc/sys/kernel/core_pattern ] && {
                procd_set_param limits core="unlimited"
        }
        procd_close_instance
}

4.coredump调试方式

编译完成之后会生生成root-debug-1.0.2.0-ZHA0103.tgz,解压缩

14.07/staging_dir/toolchain-mipsel_1004kc_gcc-4.8-linaro_uClibc-0.9.33.2/bin/mipse
l-openwrt-linux-gdb
set solib-absolute-prefix debug-root
#设置debug root(带有debug信息的程序)
file bin/ash
#load出现coredump的应用用程序
core ash.xxxx.core #load core文文件
bt #dump调用用栈

如果没有debug-root,则使用用以下方方法调试(可执行行行程序需要带上-g编译)

14.07/staging_dir/toolchain-mipsel_1004kc_gcc-4.8-linaro_uClibc-0.9.33.2/bin/mipse
l-openwrt-linux-gdb filename core
bt #dump调用用栈

file：源文件
core：coredump文件

/home/linye/ziroom/ZMAX/ZHA0107/zrouter/14.07/staging_dir/toolchain-mipsel_1004kc_gcc-4.8-linaro_uClibc-0.9.33.2/bin/mipsel-openwrt-linux-gdb /home/linye/ziroom/ZGATEWAY/git/zgateway/out/ZHA0107/M04C02D02Z02L02/zgateway/ZGateway ZGateway.1621223290.27914.11.core

如下：

linye@linye-ubuntu:~/ziroom/ZMAX/ZHA0103/coredump$ /home/linye/ziroom/ZMAX/ZHA0107/zrouter/14.07/staging_dir/toolchain-mipsel_1004kc_gcc-4.8-linaro_uClibc-0.9.33.2/bin/mipsel-openwrt-linux-gdb 

GNU gdb (Linaro GDB) 7.6-2013.05
Copyright (C) 2013 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "--host=x86_64-linux-gnu --target=mipsel-openwrt-linux-uclibc".
For bug reporting instructions, please see:
<http://bugs.launchpad.net/gdb-linaro/>.


(gdb) set solib-absolute-prefix /home/linye/ziroom/ZMAX/ZHA0107/zrouter/14.07/build_dir/target-mipsel_1004kc_uClibc-0.9.33.2/root-debug-mtk/

(gdb) file /home/linye/ziroom/ZMAX/ZHA0107/zrouter/14.07/build_dir/target-mipsel_1004kc_uClibc-0.9.33.2/root-debug-mtk/usr/sbin/zgateway_test


Reading symbols from /home/linye/ziroom/ZMAX/ZHA0103/coredump/AmberGwZ3...done.

(gdb) core zgateway_test.1605772919.4535.11.core 
[New LWP 4537]
[New LWP 4535]
[New LWP 4536]
Core was generated by `zgateway_test -d4'.
Program terminated with signal 11, Segmentation fault.
#0  0x77221f64 in free (mem=0xc29ca0) at libc/stdlib/malloc-standard/free.c:326
326         unlink(p, bck, fwd);
(gdb) bt
#0  0x77221f64 in free (mem=0xc29ca0) at libc/stdlib/malloc-standard/free.c:326
#1  0x00406894 in clean () at connector/http/src/http_curl.c:84
#2  0x00406efc in CLibcurl_post (eventType=<optimized out>, 
    pUrl=0x76fc5ed0 "https://gw.zihome.com/v1/gateway/register", 
    pData=0xc0d458 "{\n\t\"mac\":\t\"DC4BDD1DFA48\",\n\t\"gatewayVersion\":\t\"2.1.5\",\n\t\"model\":\t\"M04C02D02Z02L01\"\n}")
    at connector/http/src/http_curl.c:152
#3  0x00407318 in requestGatewayRegister ()
    at connector/http/src/http_manager.c:62
#4  0x004077f4 in run () at connector/http/src/http_manager.c:178
#5  http_manager_thread (arg=<optimized out>)
    at connector/http/src/http_manager.c:242
#6  0x774fad70 in start_thread (arg=0x76fc6530)
    at libpthread/nptl/pthread_create.c:297
#7  0x774f30f0 in __thread_start () at ./libc/sysdeps/linux/mips/clone.S:146
Backtrace stopped: frame did not save the PC

5 在线gdb运行

• 1.gdb file(未经过strip的文件)
• 2.run args(运行参数)
• 3.ctrl+d退出

root@zihome:/zihome/plugins/zgateway# gdb AmberGwZ3-nostrip 
GNU gdb (GDB) 7.5
Copyright (C) 2012 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "mipsel-openwrt-linux".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /zihome/plugins/zgateway/AmberGwZ3-nostrip...done.
(gdb) run -n1 -p/dev/ttyS0 -b115200 -w /zihome/plugins/zgateway/data/dusun/amber -d
Starting program: /zihome/plugins/zgateway/AmberGwZ3-nostrip -n1 -p/dev/ttyS0 -b115200 -w /zihome/plugins/zgateway/data/dusun/amber -d
warning: no loadable sections found in added symbol-file /lib/ld-uClibc.so.0
warning: Unable to find dynamic linker breakpoint function.
GDB will be unable to debug shared library initializers
and track explicitly loaded dynamic code.
warning: no loadable sections found in added symbol-file /lib/libm.so.0
warning: no loadable sections found in added symbol-file /lib/libpthread.so.0
warning: File "/lib/libthread_db.so.1" auto-loading has been declined by your `auto-load safe-path' set to "$debugdir:$datadir/auto-load".
warning: Unable to find libthread_db matching inferior's thread library, thread debugging will not be available.
warning: no loadable sections found in added symbol-file /lib/libubus.so
warning: no loadable sections found in added symbol-file /lib/libubox.so
warning: no loadable sections found in added symbol-file /lib/libblobmsg_json.so
warning: no loadable sections found in added symbol-file /usr/lib/libjson-c.so.2
warning: no loadable sections found in added symbol-file /usr/lib/libncurses.so.5
warning: no loadable sections found in added symbol-file /lib/libgcc_s.so.1
warning: no loadable sections found in added symbol-file /lib/libc.so.0
warning: no loadable sections found in added symbol-file /lib/libdl.so.0

signal 6 - SIGABRT
free 多次

char *p = malloc(100);
free(p);
free(p);

fclose 多次
// fclose 内部调用 free

FILE *fp = fopen("test.txt", "wb+");
printf("%p\n", fp);
fclose(fp);
printf("%p\n", fp);
fclose(fp);
printf("%p\n", fp);

查看栈回溯，跟 free 多次一样，查看 glibc/libio

signal 11 - SIGSEGV
引用空指针成员

struct Hello *p = NULL;
printf("%d\n", p->a);

fclose 空指针
// fclose 内部引用 FILE 成员

FILE *fp = NULL;
fclose(fp);

跟引用空指针成员一致。