Issue with Sysdig when running in Kubernetes where Cilium is also installed.

[saleem-accuknox]

When I try to run Sysdig chisel in Kubernetes (GKE) with Cilium also installed, I get the Segmentation Fault:

But, when I run it without Cilium, I do not have any issues running it.
Command used to run Sydig Chisel:
/usr/bin/sysdig -k http://127.0.0.1:8080 -c chisel config.yaml

The chisel

-- Chisel description
description = ""
short_description = "Monitor syscalls"
category = ""

-- Chisel argument list: Invoke as sudo sysdig -c chisel/path/to/config.yaml
args =
{
	{
		name = "config_file",
		description = "The path to the configuration yaml file",
		argtype = "string"
	}
}

-- Imports
--local inspect = require('inspect')
--local json = require ("dkjson")
local pb = require "pb"
local protoc = require "protoc"
local yaml = require('yaml')
local zmq = require "lzmq"

-- List of all field names
local all_field_names = {}
-- Map of field names to handle
local field_handles = {}
-- Map of field names to split fields (e.g. field_name = "evt.name" -> { "evt", "name" })
local split_fields = {}
-- List of common field names
--local common_field_names = {}

-- ZMQ context and socket
local context
local socket

-- Argument notification callback
function on_set_arg(name, val)
	if name == "config_file" then
		config_file = val
		return true
	end
	return false
end

-- Capture start
function on_capture_start()
	return true
end

-- Initialization callback
function on_init()
	-- read configuration file
	local fd = assert(io.open(config_file, "r"))
	local content = fd:read("*all")
	fd:close()

	-- parse configuration
	local config = yaml.eval(content)
	--print(inspect(config))

	-- make sure we have required fields in the config
	assert(config.containersec.addr)
	assert(config.containersec.schema)
	assert(config.containersec.filter_suffix)
	assert(config.containersec.field_sets)
	assert(config.containersec.common_fields)
	assert(config.containersec.syscalls)
	assert(config.containersec.deployment_type)

	-- read .proto schema file (assuming you are running from project root)
	print("Loading protobuf schema: " .. config.containersec.schema)
	fd = assert(io.open(config.containersec.schema, "r"))
	content = fd:read("*all")
	fd:close()

	-- load the schema
	assert(protoc:load(content))

	-- Connect to message queue
	context = zmq.context()
	socket, err = context:socket(zmq.PUB)
	print("Binding to " .. config.containersec.addr)
	socket:bind(config.containersec.addr)

	-- Get all field names
	local index = 1
	for field_set, field_list in pairs(config.containersec.field_sets) do
		for i, field_name in ipairs(field_list) do
			all_field_names[index] = field_name
			index = index + 1
			field_handles[field_name] = chisel.request_field(field_name)
			-- Split the field name on the first period
			-- k8s.pod.name -> k8s, pod.name
			local s, e = string.find(field_name, '.', 1, true)
			-- k8s
			local category = string.sub(field_name, 1, s-1)
			-- pod.name
			local sub_field = string.sub(field_name, e+1, string.len(field_name))
			-- Replace remaining periods with underscore in fieldname
			-- pod_name
			sub_field = sub_field:gsub("%.", "_")
			-- [k8s.pod.name] = { k8s, pod_name }
			split_fields[field_name] = {category, sub_field}
		end
	end

	-- Set the filter
	local filter = "evt.type in (" .. table.concat(config.containersec.syscalls, ",") .. ") and container.id != host" .. config.containersec.filter_suffix
	--print("Filter: " .. filter)
	chisel.set_filter(filter)

	return true
end

function handle_event(evt)
	-- Create the event
	local event = {}

	-- Get values for fields
	for i, field_name in ipairs(all_field_names) do
		-- k8s.pod.name
		local field_handle = field_handles[field_name]
		local value = evt.field(field_handle)
		if value ~= nil then
			-- Get the split field names (e.g { "evt", "name" }
			-- { k8s, pod_name }
			local tokens = split_fields[field_name]
			-- Get the first and second parts
			-- k8s
			local first = tokens[1]
			-- pod_name
			local second = tokens[2]
			-- Make sure the first table exists
			if event[first] == nil then
				event[first] = {}
			end
			-- Strip brackets from field name (e.g. arg[0] -> arg0)
			second = string.gsub(second, "%[", "")
			second = string.gsub(second, "%]", "")
			-- Add the field value to event
			event[first][second] = value
		end
	end

	-- send event to message queue
	local data = assert(pb.encode("proto.Message", event))
	if socket ~= nil and not socket:closed() then
		socket:send("sysdig", zmq.SNDMORE)
		socket:send(data)
	end
end

-- Event parsing callback
function on_event()
	local status, err = pcall(handle_event, evt);

	if not status then
		print("handle_event failed: " .. err)
	end

	return true
end

-- End of capture callback
function on_capture_end()
	-- Close socket and terminate context
	socket:close()
	context:term()

	return true
end

The config file:

containersec:
  deployment_type: kubernetes # kubernetes or container
  # ZMQ bind address
  addr: tcp://*:3654
  # Path to protobuf message schema
  schema: /message.proto
  # Filter suffix
  filter_suffix: " and evt.dir = <"
  # Named field classes
  field_sets:
    kubernetes_fields:
      - k8s.pod.name
      - k8s.ns.name
      - k8s.pod.labels
    container_fields:
      - container.id
      - container.name
    evt_fields:
      - evt.args
      - evt.arg[0]
      - evt.arg[1]
      - evt.arg[2]
      - evt.arg[3]
      - evt.category
      - evt.count
      - evt.cpu
      - evt.datetime
      - evt.deltatime
      - evt.deltatime.ns
      - evt.deltatime.s
      - evt.dir
      - evt.failed
      - evt.info
      - evt.is_io
      - evt.is_io_read
      - evt.is_io_write
      - evt.is_open_read
      - evt.is_open_write
      - evt.is_syslog
      - evt.is_wait
      - evt.latency
      - evt.latency.human
      - evt.latency.ns
      - evt.latency.s
      - evt.num
      - evt.outputtime
      - evt.rawarg.res
      - evt.rawtime
      - evt.rawtime.ns
      - evt.rawtime.s
      - evt.reltime
      - evt.reltime.ns
      - evt.reltime.s
      - evt.res
      - evt.rawres
      - evt.time
      - evt.time.iso8601
      - evt.time.s
      - evt.type
    proc_fields:
      - proc.aname
      - proc.apid
      - proc.args
      - proc.cmdline
      - proc.cwd
      - proc.duration
      - proc.env
      - proc.exe
      - proc.exeline
      - proc.exepath
      - proc.fdlimit
      - proc.fdopencount
      - proc.fdusage
      - proc.is_container_healthcheck
      - proc.is_container_liveness_probe
      - proc.is_container_readiness_probe
      - proc.loginshellid
      - proc.name
      - proc.nchilds
      - proc.nthreads
      - proc.pcmdline
      - proc.pid
      - proc.pname
      - proc.ppid
      - proc.sid
      - proc.sname
      - proc.tty
      - proc.vmrss
      - proc.vmsize
      - proc.vmswap
      - proc.vpgid
      - proc.vpid
    thread_fields:
      - thread.cgroups
      - thread.exectime
      - thread.ismain
      - thread.pfmajor
      - thread.pfminor
      - thread.tid
      - thread.totexectime
      - thread.vmrss
      - thread.vmsize
      - thread.vtid
    syscall_fields:
      - syscall.type
    fd_fields:
      - fd.cip
      - fd.connected
      - fd.containerdirectory
      - fd.containername
      - fd.cport
      - fd.cproto
      - fd.dev
      - fd.dev.major
      - fd.dev.minor
      - fd.directory
      - fd.filename
      - fd.is_server
      - fd.l4proto
      - fd.lip
      - fd.lport
      - fd.lproto
      - fd.name
      - fd.name_changed
      - fd.num
      - fd.rip
      - fd.rport
      - fd.rproto
      - fd.sip
      - fd.sockfamily
      - fd.sport
      - fd.sproto
      - fd.type
      - fd.typechar
      - fd.uid
    user_fields:
      - user.uid
      - user.name
      - user.loginuid
    group_fields:
      - group.gid
      - group.name
  # List of field classes common to all system calls
  common_fields:
    - event_fields
    - proc_fields
    - syscall_fields
    - thread_fields
    - user_fields
    - group_fields
  # List of all system calls to use
  syscalls:
    - _sysctl
    - accept
    - accept4
    - arch_prctl
    - bind
    - bpf
    - capset
    - chmod
    - chown
    - chroot
    - clock_settime
    - clone
    - clone3
    - close
    - connect
    - creat
    - create_module
    - delete_module
    - dup
    - dup2
    - dup3
    - execve
    - execve
    - execveat
    - execveat
    - exit
    - fchmod
    - fchmodat
    - fchown
    - fchownat
    - fork
    - fsconfig
    - fsmount
    - fsopen
    - futimesat
    - init_module
    - ioctl
    - kexec_file_load
    - kexec_load
    - kexec_load
    - kill
    - lchown
    - link
    - linkat
    - listen
    - mkdir
    - mkdirat
    - mknod
    - mknodat
    - mount
    - move_mount
    - nfsservctl
    - open
    - openat
    - pivot_root
    - prctl
    - process_vm_readv
    - process_vm_readv
    - process_vm_writev
    - process_vm_writev
    - ptrace
    - reboot
    - recvfrom
    - recvmsg
    - recvmmsg
    - rename
    - renameat
    - renameat2
    - rmdir
    - sendto
    - sendmsg
    - sendmmsg
    - setdomainname
    - setfsgid
    - setfsuid
    - setgid
    - setgroups
    - sethostname
    - setpgid
    - setregid
    - setresgid
    - setresuid
    - setreuid
    - setsid
    - settimeofday
    - setuid
    - shutdown
    - socket
    - symlink
    - symlinkat
    - syslog
    - umount
    - umount2
    - unlink
    - unlinkat
    - userfaultfd
    - utime
    - utimensat
    - utimes
    - vfork
    - vserver

[nyrahul]

The issue can be produced with two simple steps:

1. execute: sudo sysdig syscall.type=bpf and evt.res!=nil
2. execute any code that loads bpf code ... use examples from libbpf/bootstrap.

The problem is with evt.res handling of bpf syscall in sysdig. If the evt.res clause is removed then there is no core dump.

Since cilium agent pods use bpf syscall, the issue is seen in the context.

[github-actions]

This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.