Loki

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

cd /tmp
yum -y install unzip curl
curl -O -L "https://ghproxy.com/https://github.com/grafana/loki/releases/download/v2.3.0/loki-linux-amd64.zip"
unzip "loki-linux-amd64.zip"
sudo cp loki-linux-amd64 /usr/local/bin/
sudo chmod u+x /usr/local/bin/loki-linux-amd64
mkdir -p /etc/loki
mkdir -p /data/loki
cat << EOF > /etc/loki/loki-local-config.yaml
auth_enabled: false
server:
  http_listen_port: 3100
ingester:
  lifecycler:
    address: 127.0.0.1
    ring:
      kvstore:
        store: inmemory
      replication_factor: 1
    final_sleep: 0s
  chunk_idle_period: 5m
  chunk_retain_period: 30s
  max_transfer_retries: 0
schema_config:
  configs:
    - from: 2021-09-14
      store: boltdb
      object_store: filesystem
      schema: v11
      index:
        prefix: index_
        period: 168h
storage_config:
  boltdb:
    directory: /data/loki/index
  filesystem:
    directory: /data/loki/chunks
limits_config:
  enforce_metric_name: false
  reject_old_samples: true
  reject_old_samples_max_age: 168h
chunk_store_config:
  max_look_back_period: 0s
table_manager:
  retention_deletes_enabled: false
  retention_period: 0s
EOF
cat << EOF > /etc/systemd/system/lokid.service
[Unit]
Description=lokid service node
After=network-online.target
[Service]
Type=simple
ExecStart=/usr/local/bin/loki-linux-amd64  --config.file=/etc/loki/loki-local-config.yaml
Restart=always
RestartSec=30s
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload && systemctl enable lokid && systemctl start lokid

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

cd /tmp
curl -O -L "https://ghproxy.com/https://github.com/grafana/loki/releases/download/v2.3.0/promtail-linux-amd64.zip"
unzip "promtail-linux-amd64.zip"
sudo cp promtail-linux-amd64 /usr/local/bin/
sudo chmod u+x /usr/local/bin/promtail-linux-amd64
mkdir -p /etc/promtail
cd /etc/promtail
wget https://raw.githubusercontent.com/grafana/loki/main/clients/cmd/promtail/promtail-local-config.yaml
cat > /etc/systemd/system/promtail.service <<EOF
[Unit]
Description=Promtail service
After=network.target
[Service]
Type=simple
ExecStart=/usr/local/bin/promtail-linux-amd64 --config.file /etc/promtail/promtail-local-config.yaml
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload && systemctl enable promtail && systemctl start promtail

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

cd /tmp
curl -O -L "https://ghproxy.com/https://github.com/grafana/loki/releases/download/v2.3.0/logcli-linux-amd64.zip"
unzip "logcli-linux-amd64.zip"
sudo cp logcli-linux-amd64 /usr/local/bin/
sudo chmod u+x /usr/local/bin/logcli-linux-amd64
[root@xxx promtail]# logcli-linux-amd64 labels job
http://localhost:3100/loki/api/v1/label/job/values?end=1631585752373078818&start=1631582152373078818
varlogs
[root@cf-prod-ops promtail]# logcli-linux-amd64 query '{job="varlogs"}'
http://localhost:3100/loki/api/v1/query_range?direction=BACKWARD&end=1631585815853130831&limit=30&query=%7Bjob%3D%22varlogs%22%7D&start=1631582215853130831
Common labels: {filename="/var/log/cloud-init.log", job="varlogs"}
2021-09-14T10:08:51+08:00 {} 2021-09-01 09:43:13,281 - handlers.py[DEBUG]: finish: modules-final: SUCCESS: running modules for final
2021-09-14T10:08:51+08:00 {} 2021-09-01 09:43:13,281 - util.py[DEBUG]: cloud-init mode 'modules' took 0.199 seconds (0.19)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

# GET /ready
curl localhost:3100/ready
ready

# GET /metrics
curl -s localhost:3100/metrics|head -n 5
# HELP cortex_cache_corrupt_chunks_total Total count of corrupt chunks found in cache.
# TYPE cortex_cache_corrupt_chunks_total counter
cortex_cache_corrupt_chunks_total 0
# HELP cortex_chunk_store_chunks_per_query Distribution of #chunks per query.
# TYPE cortex_chunk_store_chunks_per_query histogram

#GET /config
curl -s localhost:3100/config|head -n 5
target: all
http_prefix: ""
server:
  http_listen_address: ""
  http_listen_port: 3100

#GET /loki/api/v1/status/buildinfo
curl -s localhost:3100/loki/api/v1/status/buildinfo
{"version":"","revision":"","branch":"","buildUser":"","buildDate":"","goVersion":""}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

Loki Cheat Sheet
See your logs
Start by selecting a log stream from the Log browser, or alternatively you can write a stream selector into the query field.
Here is an example of a log stream:
{job="default/prometheus"}
Combine stream selectors
{app="cassandra",namespace="prod"}
Returns all log lines from streams that have both labels.
Filtering for search terms.
{app="cassandra"} |~ "(duration|latency)s*(=|is|of)s*[d.]+"
{app="cassandra"} |= "exact match"
{app="cassandra"} != "do not match"
LogQL supports exact and regular expression filters.
Log pipeline
{job="mysql"} |= "metrics" | logfmt | duration > 10s
This query targets the MySQL job, filters out logs that don’t contain the word "metrics" and parses each log line to extract more labels and filters with them.
Count over time
count_over_time({job="mysql"}[5m])
This query counts all the log lines within the last five minutes for the MySQL job.
Rate
rate(({job="mysql"} |= "error" != "timeout")[10s])
This query gets the per-second rate of all non-timeout errors within the last ten seconds for the MySQL job.
Aggregate, count, and group
sum(count_over_time({job="mysql"}[5m])) by (level)
Get the count of logs during the last five minutes, grouping by level.