Monitoring Server and Alerting with Grafana and Prometheus

$ ifconfig

# copy local ip address 4 at eno# section 
# Put it into targets at prometheus.yml

$ tree
.
├── docker-compose.yml
├── grafana.ini
└── prometheus.yml

$ docker-compose up -d

$ docker-compose ps
              Name                            Command               State                    Ports                  
--------------------------------------------------------------------------------------------------------------------
servermonitoring_grafana_1         /run.sh                          Up      0.0.0.0:3000->3000/tcp,:::3000->3000/tcp
servermonitoring_node-exporter_1   /bin/node_exporter               Up      0.0.0.0:9100->9100/tcp,:::9100->9100/tcp
servermonitoring_prometheus_1      /bin/prometheus --config.f ...   Up      0.0.0.0:9090->9090/tcp,:::9090->9090/tcp

# Find Grafana container
$ docker ps -a

$ sudo docker cp grafana_conitaner_id:/etc/grafana/grafana.ini ./
$ sudo vi grafana.ini
$ sudo docker cp ./grafana.ini grafana_conitaner_id:/etc/grafana/grafana.ini

#################################### SMTP / Emailing ##########################
[smtp]
;enabled = false
;host = localhost:25
;user =
# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;"""
;password =
;cert_file =
;key_file =
;skip_verify = false
;from_address = admin@grafana.localhost
;from_name = Grafana
# EHLO identity in SMTP dialog (defaults to instance_name)
;ehlo_identity = dashboard.example.com
# SMTP startTLS policy (defaults to 'OpportunisticStartTLS')
;startTLS_policy = NoStartTLS
# Enable trace propagation in e-mail headers, using the 'traceparent', 'tracestate' and (optionally) 'baggage' fields (defaults to false)
;enable_tracing = false

[smtp.static_headers]
# Include custom static headers in all outgoing emails
;Foo-Header = bar
;Foo = bar

[emails]
;welcome_email_on_sign_up = false
;templates_pattern = emails/*.html, emails/*.txt
;content_types = text/html

#################################### SMTP / Emailing ##########################
[smtp]
enabled = true
host = smtp.gmail.com:587
user = example@gmail.com
# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;"""
password = """your_pw"""
;cert_file =
;key_file =
skip_verify = false
from_address = example@gmail.com
from_name = Grafana

grafana:
    image: grafana/grafana
    volumes:
      - ./grafana.ini:/etc/grafana/grafana.ini

$ docker-compose down
$ docker-compose up -d
$ docker-compose exec grafana cat /etc/grafana/grafana.ini | grep -A 20 "\[smtp\]"

/**
 * Copyright 2024 Ryo M
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
version: '3'
services:
  prometheus:
    image: prom/prometheus
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
    ports:
      - '9090:9090'
  grafana:
    image: grafana/grafana
    ports:
      - "3000:3000"
  node-exporter:
    image: quay.io/prometheus/node-exporter
    ports:
      - 9100:9100
    volumes:
      - ./proc:/host/proc
      - ./sys:/host/sys
      - ./rootfs:/rootfs

/**
 * Copyright 2024 Ryo M
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
global:
  scrape_interval: 15s
  external_labels:
    monitor: 'codelab-monitor'
scrape_configs:
  - job_name: 'node'
    scrape_interval: 5s
    static_configs:
      - targets: ['your_local_ip:9100']