loongcollector#
LoongCollector 是一个开源的分布式日志采集系统，基于 Fluentd 构建，支持多语言，多平台，多协议，多数据源，多数据目的。
eBPF + 采集#
需要自行编译，github的release有问题，coolbpf的分支错误，需要将其切换到loongcollector-base
# 编译镜像 sls-opensource-registry.cn-shanghai.cr.aliyuncs.com/loongcollector-community-edition/loongcollector-build-linux:2.1.13
# 更新core/_third_party/
# 其中coolbpf要是loongcollector-base分支编译出的eBPF才能用
# under /core/build
cmake ..
make -sj$(nproc)
简单示例#
enable: true
inputs:
- Type: input_network_observer
ProbeConfig:
L7Config:
Enable: true
SampleRate: 1.0
EnableLog: true
EnableMetric: true
EnableSpan: true
L4Config:
Enable: true
ApmConfig:
Workspace: prod
AppName: test
AppId: xx@test
ServiceId: ss@test
Language: java
EnableLog: true
EnableMetric: true
EnableSpan: true
EnableProtocols:
- "http"
DisableConnStats: false
EnableConnTrackerDump: true
flushers:
- Type: flusher_file
FilePath: input_network_observer.log输出拆分metric、trace\log#
enable: true
inputs:
- Type: input_network_observer
ProbeConfig:
L7Config:
Enable: true
SampleRate: 1.0
EnableLog: true
EnableMetric: true
EnableSpan: true
L4Config:
Enable: true
ApmConfig:
Workspace: prod
AppName: test
AppId: xx@test
ServiceId: ss@test
Language: java
EnableLog: true
EnableMetric: true
EnableSpan: true
EnableProtocols:
- "http"
DisableConnStats: false
EnableConnTrackerDump: true
# 多路输出
flushers:
- Type: flusher_file
FilePath: input_network_observer_metric.log
Match:
Type: event_type
Value: metric
- Type: flusher_file
FilePath: input_network_observer_trace.log
Match:
Type: event_type
Value: trace
- Type: flusher_file
FilePath: input_network_observer_log.log
Match:
Type: event_type
Value: logmetric#
{
"__host_ip__": "192.168.124.50",
"__hostname__": "localhost.localdomain",
"__machine_uuid__": "0FAC399A-EC91-11F0-A1B4-3D6692D60605",
"__source__": "192.168.124.50",
"acs_arms_service_id": "ss@test",
"acs_cms_workspace": "prod",
"data_type": "metric",
"host": "",
"language": "java",
"pid": "xx@test",
"serverIp": "192.168.124.50",
"service": "test",
"source": "apm",
"technology": "ebpf",
"__time__": 1767877042,
"__labels__": {
"status": "2xx",
"workloadName": "",
"workloadKind": "",
"namespace": "",
"rpc": "/conf/",
"rpcType": "0",
"callType": "http",
"callKind": "http",
"endpoint": "/conf/",
"destId": "192.168.124.1"
},
"__name__": "arms_rpc_requests_by_status_count",
"__value__": 1.0
}trace#
{
"__host_ip__": "192.168.124.50",
"__hostname__": "localhost.localdomain",
"__machine_uuid__": "0FAC399A-EC91-11F0-A1B4-3D6692D60605",
"__source__": "192.168.124.50",
"acs.arms.service.id": "ss@test",
"acs.cms.workspace": "prod",
"app": "",
"arms.app.type": "apm",
"arms.appId": "xx@test",
"data_type": "trace",
"host.ip": "192.168.124.50",
"host.name": "",
"language": "java",
"service.name": "test",
"technology": "ebpf",
"__time__": 1767877035,
"traceId": "f2d3a99bb387e089a5d223ee96ec44fa79bc7130c7af716c65e6658e1cd6f6c0",
"spanId": "8eff0a1008f9bfddada56cf0d2ac9f0e",
"parentSpanId": "",
"spanName": "/conf/continuous_pipeline_config/common_v2/",
"startTime": 1767877035774786053,
"endTime": 1767877035775402006,
"duration": 615953,
"attributes": {
"host": "192.168.124.50",
"host.name": "",
"k8s.pod.name": "",
"k8s.pod.ip": "",
"k8s.namespace": "",
"k8s.workload.kind": "",
"k8s.workload.name": "",
"k8s.peer.pod.name": "external",
"k8s.peer.pod.ip": "external",
"k8s.peer.namespace": "external",
"k8s.peer.workload.kind": "external",
"peerWorkloadName": "external",
"k8s.peer.service.name": "external",
"protocol": "ProtoHTTP",
"local.addr": "192.168.124.50:16415",
"remote.addr": "192.168.124.1:30352",
"remote.port": "30352",
"rpcType": "0",
"call.kind": "http",
"call.type": "http",
"destId": "192.168.124.1",
"endpoint": "",
"fd": "4",
"process.pid": "9733",
"container.id": "",
"trace.role": "IsServer",
"ip": "192.168.124.50",
"remote.ip": "192.168.124.1",
"net.namespace": "-268435304",
"family": "AF_INET",
"startTsNs": "3218715424237",
"rpc": "/conf/continuous_pipeline_config/common_v2/",
"req.body": "",
"resp.body": "",
"http.req.body.size": "0",
"http.resp.body.size": "0",
"version": "http1.1"
},
"scope": {}
}log#
{
"__host_ip__": "192.168.124.50",
"__hostname__": "localhost.localdomain",
"__machine_uuid__": "0FAC399A-EC91-11F0-A1B4-3D6692D60605",
"__source__": "192.168.124.50",
"data_type": "log",
"__time__": 1767877035,
"k8s.peer.pod.name": "external",
"k8s.peer.pod.ip": "external",
"k8s.peer.namespace": "external",
"k8s.peer.workload.kind": "external",
"k8s.peer.workload.name": "external",
"k8s.peer.service.name": "external",
"protocol": "ProtoHTTP",
"local.addr": "192.168.124.50:16415",
"remote.addr": "192.168.124.1:30352",
"remote.port": "30352",
"rpc_type": "0",
"arms.call.kind": "http",
"arms.call.type": "http",
"dest.id": "192.168.124.1",
"fd": "4",
"pid": "9733",
"trace.role": "IsServer",
"ip": "192.168.124.50",
"remote.ip": "192.168.124.1",
"netns": "-268435304",
"family": "AF_INET",
"start_time_nsec": "3218715424237",
"latency": "615953.000000",
"http.method": "GET",
"http.path": "/conf/continuous_pipeline_config/common_v2/",
"http.version": "http1.1",
"http.status.code": "200",
"http.req.body": "",
"http.resp.body": ""
}ebpf => VictoriaMetrics#
enable: true
inputs:
- Type: input_file
FilePaths:
- input_network_observer_log.log
flushers:
- Type: flusher_loki
URL: http://192.168.124.50:9428/insert/loki/api/v1/push
MaxMessageWait: 100000000
MaxMessageBytes: 16
Timeout: 1000000000000
MinBackoff: 100000000000
MaxBackoff: 1000000000000
MaxRetries: 10
StaticLabels:
source: ilogtail