Configuration

Configuration Grafana's configuration primarily resides in the grafana.ini file, typically located in the /etc/grafana/ directory. Key sections include:

[server]: Defines server properties like the HTTP port and domain.
[database]: Configures the database used for storing settings and dashboards.
[security]: Settings related to security, such as admin user, password, and cookie settings.
[users]: Defines user settings, including invite expiration and viewing limits.

Install Node Exporter

mkdir prometheus
cd prometheus/
wget https://github.com/prometheus/node_exporter/releases/download/v1.7.0/node_exporter-1.7.0.linux-amd64.tar.gz 
tar -xvzf node_exporter-1.7.0.linux-amd64.tar.gz
mv node_exporter-1.7.0.linux-amd64/node_exporter /usr/local/bin/

Configure Node Exporter

vi /etc/nodeexporter/config.yaml

Add the following lines

tls_server_config:
  cert_file: /etc/nodeexporter/certificates.crt
  key_file: /etc/nodeexporter/risecorp.key

basic_auth_users:
  admin: $2y$12$rGDWo4Q6p6CJVHbLDx7sYOMdZ8nhJjMWLJREM4vcxaWC7ZXlLqSWm

Create Node Exporter Service

vi /etc/systemd/system/nodeexporter.service

Add the following lines

[Unit]
Description=Node Exporter
After=network.target

[Service]
User=root
Group=root
Type=simple
ExecStart=/usr/local/bin/node_exporter --web.config.file=/etc/nodeexporter/config.yaml

[Install]
WantedBy=multi-user.target

Enable and start the service

systemctl daemon-reload
systemctl start nodeexporter
systemctl enable nodeexporter

Install Prometheus

wget https://github.com/prometheus/prometheus/releases/download/v2.50.1/prometheus-2.50.1.linux-amd64.tar.gz 
tar -xvzf prometheus-2.50.1.linux-amd64.tar.gz
mv prometheus-2.50.1.linux-amd64/prometheus /usr/local/bin/
mv prometheus-2.50.1.linux-amd64/promtool /usr/local/bin/
mkdir -p /etc/prometheus /var/lib/prometheus
mv prometheus-2.50.1.linux-amd64/console* /etc/prometheus/

Configure Prometheus

vi /etc/prometheus/prometheus.yml

Add the following lines

global:
  scrape_interval: 10s

scrape_configs:

job_name: 'prometheus_metrics'
scrape_interval: 5s
static_configs:

targets: ['localhost:9090']

job_name: 'node_exporter_metrics'
scheme: https
tls_config:
  ca_file: /etc/prometheus/certificates.crt
  insecure_skip_verify: true
basic_auth:
  username: admin
  password: Rise@321
scrape_interval: 5s
static_configs:

targets: ['localhost:9100']

vi /etc/prometheus/web.yaml

Add the following lines

tls_server_config:
  cert_file: /etc/prometheus/certificates.crt
  key_file: /etc/prometheus/risecorp.key

http_server_config:
  http2: true

basic_auth_users:
  admin: $2y$12$PXoepnvlnmZ.jz8bh8ISb.qK45Qk.wFzeav1OO9ZpzftRYOCg5Hp6

Create Prometheus Service
vi /etc/systemd/system/prometheus.service

Add the following lines

[Unit]
Description=Prometheus
After=network.target

[Service]
User=root
Group=root
Type=simple
ExecStart=/usr/local/bin/prometheus --config.file /etc/prometheus/prometheus.yml --storage.tsdb.path /var/lib/prometheus/ --web.console.templates=/etc/prometheus/consoles --web.console.libraries=/etc/prometheus/console_libraries --web.config.file=/etc/prometheus/web.yaml

[Install]
WantedBy=multi-user.target

Enable and start the service

systemctl daemon-reload
systemctl start prometheus
systemctl enable prometheus

Promtail Setup
Install Promtail
wget https://github.com/grafana/loki/releases/download/v2.9.5/promtail-linux-amd64.zip 
unzip promtail-linux-amd64.zip
chmod +x promtail-linux-amd64
mv promtail-linux-amd64 /usr/local/bin/

Configure Promtail
vi /root/loki/promtailconfig.yaml

Add the following lines

server:
  http_listen_port: 9080
  grpc_listen_port: 0  # Set to 0 to disable gRPC server or assign another port if needed.

positions:
  filename: /tmp/positions.yaml  # File to store the positions of logs being read.

clients:

url: 'http://localhost:3100/loki/api/v1/push'  # URL to push logs to Loki.

scrape_configs:

job_name: system
static_configs:

targets: ['localhost']  # List format is required.
labels:
  job: varlogs
  path: /var/log/syslog  # Path to log files for this job.

job_name: emr
static_configs:

targets: ['localhost']  # List format is required.
labels:
  job: elasticsearch
  path: /var/log/elasticsearch/elasticsearch.log  # Path to log files for this job.

job_name: platform
static_configs:

targets: ['localhost']  # List format is required.
labels:
  job: platform
  path: /opt/localstorage/err.log  # Path to log files for this job.

Create Promtail Service
vi /etc/systemd/system/promtail.service

Add the following lines

[Unit]
Description=Promtail service
After=network.target

[Service]
Type=simple
User=root
ExecStart=/usr/local/bin/promtail -config.file /root/loki/promtailconfig.yaml

[Install]
WantedBy=multi-user.target

Enable and start the service

systemctl daemon-reload
systemctl start promtail
systemctl enable promtail

Loki Setup
Install Loki
mkdir loki
cd loki
wget https://github.com/grafana/loki/releases/download/v2.9.5/loki-linux-amd64.zip 
unzip loki-linux-amd64.zip
chmod +x loki-linux-amd64
mv loki-linux-amd64 /usr/local/bin/

Configure Loki
vi /root/loki/loki.yaml

Add the following lines

auth_enabled: false

server:
  http_listen_port: 3100
  grpc_listen_port: 9096

common:
  path_prefix: /mnt/logs/
  storage:
    filesystem:
      chunks_directory: /mnt/logs/chunks
      rules_directory: /mnt/logs/rules
  replication_factor: 1
  ring:
    instance_addr: 0.0.0.0
    kvstore:
      store: inmemory

schema_config:
  configs:
    - from: 2020-10-24
      store: boltdb-shipper
      object_store: filesystem
      schema: v11
      index:
        prefix: index_
        period: 24h

ruler:
  alertmanager_url: http://localhost:9093

Create Loki Service
vi /etc/systemd/system/loki.service

Add the following lines

[Unit]
Description=loki service
After=network.target

[Service]
Type=simple
User=root
ExecStart=/usr/local/bin/loki-linux-amd64 -config.file=/root/loki/loki.yaml -config.expand-env=true
Errorlog=/var/log/loki/loki_error.log
Restart=on-failure
RestartSec=5s

[Install]
WantedBy=multi-user.target

Enable and start the service

systemctl daemon-reload
systemctl start loki
systemctl enable loki

Grafana-Configuration Configuration Files Primary Configuration (grafana.ini) ini

[paths]
data = /var/lib/grafana
logs = /var/log/grafana
plugins = /var/lib/grafana/plugins
provisioning = /etc/grafana/provisioning
[server]
protocol = http
http_port = 3000
domain = grafana.yourdomain.com
root_url = https://grafana.yourdomain.com
serve_from_sub_path = false
router_logging = false
[database]
type = postgres
host = postgres:5432
name = grafana
user = grafana
password = ${GF_DATABASE_PASSWORD}
ssl_mode = disable
max_idle_conn = 2
max_open_conn = 0
conn_max_lifetime = 14400
log_queries = false
[session]
provider = file
provider_config = sessions
cookie_name = grafana_sess
cookie_secure = true
session_life_time = 86400
gc_interval_time = 86400
[security]
admin_user = admin
admin_password = ${GF_SECURITY_ADMIN_PASSWORD}
secret_key = ${GF_SECURITY_SECRET_KEY}
encryption_provider = secretKey.v1
available_encryption_providers = secretKey.v1
login_remember_days = 7
cookie_username = grafana_user
cookie_remember_name = grafana_remember
disable_gravatar = true
data_source_proxy_whitelist = 
content_security_policy = true
content_security_policy_report_only = false
strict_transport_security = true
x_content_type_options = true
x_xss_protection = true
[users]
allow_sign_up = false
allow_org_create = false
auto_assign_org = true
auto_assign_org_id = 1
auto_assign_org_role = Viewer
verify_email_enabled = false
login_hint = email or username
password_hint = password
default_theme = dark
home_page = 
external_manage_link_url = 
external_manage_link_name = 
external_manage_info = 
[auth]
login_cookie_name = grafana_session
login_maximum_inactive_lifetime_duration = 
login_maximum_lifetime_duration = 
token_rotation_interval_minutes = 10
disable_login_form = false
disable_signout_menu = false
signout_redirect_url = 
oauth_auto_login = false
oauth_skip_org_role_update_sync = false
api_key_max_seconds_to_live = -1
[auth.ldap]
enabled = false
config_file = /etc/grafana/ldap.toml
allow_sign_up = true
skip_org_role_sync = false
[unified_alerting]
enabled = true
disabled_orgs = 
min_interval = 10s
max_attempts = 3
screenshots_capture = false
screenshots_capture_timeout = 10s
screenshots_upload_timeout = 10s
[alerting]
enabled = false
execute_alerts = false
error_or_timeout = alerting
nodata_or_nullvalues = no_data
concurrent_render_limit = 5
evaluation_timeout_seconds = 30
notification_timeout_seconds = 30
max_attempts = 3
[feature_toggles]
enable = publicDashboards,correlations,canvasPanels,live,ngalert,scenes
[plugins]
enable_alpha = false
app_tls_skip_verify_insecure = false
allow_loading_unsigned_plugins = 
plugin_admin_enabled = true
plugin_admin_external_manage_enabled = false
plugin_catalog_url = https://grafana.com/api/plugins
plugin_catalog_hidden_plugins = 
[enterprise]
license_path = 
[log]
mode = console file
level = info
format = text
Environment Variables (.env)

bash

Database Configuration

GF_DATABASE_PASSWORD=secure_db_password
POSTGRES_PASSWORD=secure_db_password

Security Settings

GF_SECURITY_ADMIN_PASSWORD=secure_admin_password
GF_SECURITY_SECRET_KEY=your_very_long_secret_key_here

LDAP Configuration

GF_AUTH_LDAP_ENABLED=true
GF_AUTH_LDAP_CONFIG_FILE=/etc/grafana/ldap.toml
# Plugin Configuration
GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-simple-json-datasource,grafana-piechart-panel
# Feature Toggles
GF_FEATURE_TOGGLES_ENABLE=publicDashboards,correlations,canvasPanels,live,ngalert,scenes
# Logging
GF_LOG_LEVEL=info
GF_LOG_MODE=console file
# Server Configuration
GF_SERVER_DOMAIN=grafana.yourdomain.com
GF_SERVER_ROOT_URL=https://grafana.yourdomain.com
Data Source Provisioning
yaml

# provisioning/datasources/datasources.yml
apiVersion: 1
datasources:
  - name: Prometheus
    type: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: true
    jsonData:
      httpMethod: POST
      manageAlerts: true
      prometheusType: Prometheus
      cacheLevel: 'High'
      incrementalQuerying: true
      exemplarTraceIdDestinations:
        - name: trace_id
          datasourceUid: jaeger_uid
    secureJsonData:
      httpHeaderValue1: 'Bearer your_token_here'
  - name: InfluxDB
    type: influxdb
    access: proxy
    url: http://influxdb:8086
    database: mydb
    user: influxdb_user
    jsonData:
      httpMode: GET
      keepCookies: []
    secureJsonData:
      password: influxdb_password
  - name: PostgreSQL
    type: postgres
    access: proxy
    url: postgres:5432
    database: monitoring
    user: postgres
    jsonData:
      sslmode: disable
      maxOpenConns: 100
      maxIdleConns: 100
      connMaxLifetime: 14400
      postgresVersion: 1400
      timescaledb: false
    secureJsonData:
      password: postgres_password
  - name: Jaeger
    type: jaeger
    access: proxy
    url: http://jaeger:14268
    jsonData:
      tracesToLogs:
        datasourceUid: 'loki_uid'
        tags: ['job', 'instance', 'pod', 'namespace']
        mappedTags: [{ key: 'service.name', value: 'service' }]
        mapTagNamesEnabled: false
        spanStartTimeShift: '1h'
        spanEndTimeShift: '1h'
        filterByTraceID: false
        filterBySpanID: false
      tracesToMetrics:
        datasourceUid: 'prometheus_uid'
        tags: [{ key: 'service.name', value: 'service' }, { key: 'job' }]
        queries:
          - name: 'Sample query'
            query: 'sum(rate(traces_spanmetrics_latency_bucket{$$__tags}[5m]))'
      nodeGraph:
        enabled: true

Dashboard Provisioning yaml

# provisioning/dashboards/dashboards.yml
apiVersion: 1
providers:
  - name: 'default'
    orgId: 1
    folder: ''
    type: file
    disableDeletion: false
    updateIntervalSeconds: 10
    allowUiUpdates: true
    options:
      path: /etc/grafana/provisioning/dashboards
  - name: 'monitoring'
    orgId: 1
    folder: 'Infrastructure'
    type: file
    disableDeletion: false
    updateIntervalSeconds: 10
    allowUiUpdates: true
    options:
      path: /etc/grafana/provisioning/dashboards/monitoring
  - name: 'applications'
    orgId: 1
    folder: 'Applications'
    type: file
    disableDeletion: false
    updateIntervalSeconds: 10
    allowUiUpdates: true
    options:
      path: /etc/grafana/provisioning/dashboards/applications

Install Node Exporter​

Database Configuration

Security Settings

LDAP Configuration

Install Node Exporter