observability 之 LoongCollector

Published: at 18:16

loongcollector

LoongCollector 是一个开源的分布式日志采集系统，基于 Fluentd 构建，支持多语言，多平台，多协议，多数据源，多数据目的。

eBPF + 采集

需要自行编译，github的release有问题，coolbpf的分支错误，需要将其切换到loongcollector-base

# 编译镜像 sls-opensource-registry.cn-shanghai.cr.aliyuncs.com/loongcollector-community-edition/loongcollector-build-linux:2.1.13
# 更新core/_third_party/
# 其中coolbpf要是loongcollector-base分支编译出的eBPF才能用

# under /core/build
cmake ..
make -sj$(nproc)

简单示例

enable: true
inputs:
  - Type: input_network_observer
    ProbeConfig:
      L7Config:
        Enable: true
        SampleRate: 1.0
        EnableLog: true
        EnableMetric: true
        EnableSpan: true
      L4Config:
        Enable: true
      ApmConfig:
        Workspace: prod
        AppName: test
        AppId: xx@test
        ServiceId: ss@test
        Language: java
      EnableLog: true
      EnableMetric: true
      EnableSpan: true
      EnableProtocols:
        - "http"
      DisableConnStats: false
      EnableConnTrackerDump: true
flushers:
  - Type: flusher_file
    FilePath: input_network_observer.log

输出拆分metric、trace\log

enable: true
inputs:
  - Type: input_network_observer
    ProbeConfig:
      L7Config:
        Enable: true
        SampleRate: 1.0
        EnableLog: true
        EnableMetric: true
        EnableSpan: true
      L4Config:
        Enable: true
      ApmConfig:
        Workspace: prod
        AppName: test
        AppId: xx@test
        ServiceId: ss@test
        Language: java
      EnableLog: true
      EnableMetric: true
      EnableSpan: true
      EnableProtocols:
        - "http"
      DisableConnStats: false
      EnableConnTrackerDump: true

# 多路输出
flushers:
  - Type: flusher_file
    FilePath: input_network_observer_metric.log
    Match:
      Type: event_type
      Value: metric
  - Type: flusher_file
    FilePath: input_network_observer_trace.log
    Match:
      Type: event_type
      Value: trace
  - Type: flusher_file
    FilePath: input_network_observer_log.log
    Match:
      Type: event_type
      Value: log

metric

{
  "__host_ip__": "192.168.124.50",
  "__hostname__": "localhost.localdomain",
  "__machine_uuid__": "0FAC399A-EC91-11F0-A1B4-3D6692D60605",
  "__source__": "192.168.124.50",
  "acs_arms_service_id": "ss@test",
  "acs_cms_workspace": "prod",
  "data_type": "metric",
  "host": "",
  "language": "java",
  "pid": "xx@test",
  "serverIp": "192.168.124.50",
  "service": "test",
  "source": "apm",
  "technology": "ebpf",
  "__time__": 1767877042,
  "__labels__": {
    "status": "2xx",
    "workloadName": "",
    "workloadKind": "",
    "namespace": "",
    "rpc": "/conf/",
    "rpcType": "0",
    "callType": "http",
    "callKind": "http",
    "endpoint": "/conf/",
    "destId": "192.168.124.1"
  },
  "__name__": "arms_rpc_requests_by_status_count",
  "__value__": 1.0
}

trace

{
  "__host_ip__": "192.168.124.50",
  "__hostname__": "localhost.localdomain",
  "__machine_uuid__": "0FAC399A-EC91-11F0-A1B4-3D6692D60605",
  "__source__": "192.168.124.50",
  "acs.arms.service.id": "ss@test",
  "acs.cms.workspace": "prod",
  "app": "",
  "arms.app.type": "apm",
  "arms.appId": "xx@test",
  "data_type": "trace",
  "host.ip": "192.168.124.50",
  "host.name": "",
  "language": "java",
  "service.name": "test",
  "technology": "ebpf",
  "__time__": 1767877035,
  "traceId": "f2d3a99bb387e089a5d223ee96ec44fa79bc7130c7af716c65e6658e1cd6f6c0",
  "spanId": "8eff0a1008f9bfddada56cf0d2ac9f0e",
  "parentSpanId": "",
  "spanName": "/conf/continuous_pipeline_config/common_v2/",
  "startTime": 1767877035774786053,
  "endTime": 1767877035775402006,
  "duration": 615953,
  "attributes": {
    "host": "192.168.124.50",
    "host.name": "",
    "k8s.pod.name": "",
    "k8s.pod.ip": "",
    "k8s.namespace": "",
    "k8s.workload.kind": "",
    "k8s.workload.name": "",
    "k8s.peer.pod.name": "external",
    "k8s.peer.pod.ip": "external",
    "k8s.peer.namespace": "external",
    "k8s.peer.workload.kind": "external",
    "peerWorkloadName": "external",
    "k8s.peer.service.name": "external",
    "protocol": "ProtoHTTP",
    "local.addr": "192.168.124.50:16415",
    "remote.addr": "192.168.124.1:30352",
    "remote.port": "30352",
    "rpcType": "0",
    "call.kind": "http",
    "call.type": "http",
    "destId": "192.168.124.1",
    "endpoint": "",
    "fd": "4",
    "process.pid": "9733",
    "container.id": "",
    "trace.role": "IsServer",
    "ip": "192.168.124.50",
    "remote.ip": "192.168.124.1",
    "net.namespace": "-268435304",
    "family": "AF_INET",
    "startTsNs": "3218715424237",
    "rpc": "/conf/continuous_pipeline_config/common_v2/",
    "req.body": "",
    "resp.body": "",
    "http.req.body.size": "0",
    "http.resp.body.size": "0",
    "version": "http1.1"
  },
  "scope": {}
}

log

{
  "__host_ip__": "192.168.124.50",
  "__hostname__": "localhost.localdomain",
  "__machine_uuid__": "0FAC399A-EC91-11F0-A1B4-3D6692D60605",
  "__source__": "192.168.124.50",
  "data_type": "log",
  "__time__": 1767877035,
  "k8s.peer.pod.name": "external",
  "k8s.peer.pod.ip": "external",
  "k8s.peer.namespace": "external",
  "k8s.peer.workload.kind": "external",
  "k8s.peer.workload.name": "external",
  "k8s.peer.service.name": "external",
  "protocol": "ProtoHTTP",
  "local.addr": "192.168.124.50:16415",
  "remote.addr": "192.168.124.1:30352",
  "remote.port": "30352",
  "rpc_type": "0",
  "arms.call.kind": "http",
  "arms.call.type": "http",
  "dest.id": "192.168.124.1",
  "fd": "4",
  "pid": "9733",
  "trace.role": "IsServer",
  "ip": "192.168.124.50",
  "remote.ip": "192.168.124.1",
  "netns": "-268435304",
  "family": "AF_INET",
  "start_time_nsec": "3218715424237",
  "latency": "615953.000000",
  "http.method": "GET",
  "http.path": "/conf/continuous_pipeline_config/common_v2/",
  "http.version": "http1.1",
  "http.status.code": "200",
  "http.req.body": "",
  "http.resp.body": ""
}

ebpf => VictoriaMetrics

enable: true
inputs:
  - Type: input_file
    FilePaths:
      - input_network_observer_log.log
flushers:
  - Type: flusher_loki
    URL: http://192.168.124.50:9428/insert/loki/api/v1/push
    MaxMessageWait: 100000000
    MaxMessageBytes: 16
    Timeout: 1000000000000
    MinBackoff: 100000000000
    MaxBackoff: 1000000000000
    MaxRetries: 10
    StaticLabels:
      source: ilogtail