add role for victoriametrics v1.71.0

README.md

@@ -0,0 +1,46 @@
+Setup VictoriaMetrics 1.71.0 cluster  
+
+Example hosts file:
+
+    [vmstorage]
+    vm-1 
+    vm-2
+    vm-3
+    
+    [vminsert]
+    vm-1 
+    vm-2
+    vm-3
+    
+    [vmselect]
+    vm-1 
+    vm-2
+    vm-3
+    
+    [vmauth]
+    vm-1 
+    vm-2
+    vm-3
+    
+    [vmagent]
+    vm-1 
+    vm-2
+    vm-3
+
+Example group_vars:
+
+    victoriametrics_vmauth_users:
+      - id: "1"
+        username: "foobar"
+        password: "foobar"
+
+    victoriametrics_vmagent_prometheus_config:
+      global:
+        scrape_interval: '1m'
+        scrape_timeout: '1m'
+      scrape_configs:
+      - job_name: 'rgw'
+        stream_parse: true
+        static_configs:
+          - targets: ['127.0.0.1:9242']
+

defaults/main.yaml

@@ -0,0 +1,38 @@
+victoriametrics_corp_repo: "antonpatsev/VictoriaMetrics"
+victoriametrics_version: "1.71.0"
+
+victoriametrics_vmstorage_group: 'vmstorage'
+victoriametrics_vmstorage_version: "{{ victoriametrics_version }}"
+victoriametrics_vmstorage_data_dir: "/var/lib/victoria-metrics-cluster-data"
+
+victoriametrics_vmselect_group: 'vmselect'
+victoriametrics_vmselect_version: "{{ victoriametrics_version }}"
+
+victoriametrics_vminsert_group: 'vminsert'
+victoriametrics_vminsert_version: "{{ victoriametrics_version }}"
+victoriametrics_vminsert_replication_factor: "2"
+
+
+victoriametrics_vmauth_group: 'vmauth'
+victoriametrics_vmauth_version: "{{ victoriametrics_version }}"
+victoriametrics_vmauth_config_file: "/etc/victoriametrics/vmauth/config.yml"
+
+victoriametrics_vmauth_users: []
+#  - id: "1"
+#    username: "foobar"
+#    password: "foobar"
+
+victoriametrics_vmagent_group: 'vmagent'
+victoriametrics_vmagent_version: "{{ victoriametrics_version }}"
+victoriametrics_vmagent_promscrape_config_file: "/etc/victoriametrics/vmagent/prometheus.yml"
+victoriametrics_vmagent_remoteWrite_tmpDataPath: "/var/lib/vmagent-remotewrite-data"
+
+victoriametrics_vmagent_prometheus_config: []
+victoriametrics_vmagent_prometheus_config_default:
+  global:
+    scrape_interval: '1m'
+    scrape_timeout: '1m'
+  scrape_configs:
+    - job_name: 'vmagent'
+      static_configs:
+        - targets: ['127.0.0.1:8429']

handlers/main.yaml

@@ -0,0 +1,39 @@
+- name: Restart vmstorage
+  systemd:
+    name: vmstorage
+    state: restarted
+    enabled: yes
+    daemon_reload: yes
+  when: not vmstorage_setup.changed
+
+- name: Restart vminsert
+  systemd:
+    name: vminsert
+    state: restarted
+    enabled: yes
+    daemon_reload: yes
+  when: not vminsert_setup.changed
+
+- name: Restart vmselect
+  systemd:
+    name: vmselect
+    state: restarted
+    enabled: yes
+    daemon_reload: yes
+  when: not vmselect_setup.changed
+
+- name: Restart vmauth
+  systemd:
+    name: vmauth
+    state: restarted
+    enabled: yes
+    daemon_reload: yes
+  when: not vmauth_setup.changed
+
+- name: Restart vmagent
+  systemd:
+    name: vmagent
+    state: restarted
+    enabled: yes
+    daemon_reload: yes
+  when: not vmagent_setup.changed

meta/main.yaml

@@ -0,0 +1,7 @@
+---
+galaxy_info:
+  description: VictoriaMetrics
+  galaxy_tags:
+  - victoriametrics
+
+dependencies: []

tasks/main.yaml

@@ -0,0 +1,33 @@
+- name: Install yum corp plugin
+  dnf:
+    name: yum-plugin-copr
+    state: present
+
+- name: Check corp repo antonpatsev/VictoriaMetrics is enabled
+  shell: dnf -y copr list --enabled
+  register: corp_repo
+  changed_when: false
+
+- name: Enable corp repo antonpatsev/VictoriaMetrics
+  shell: dnf -y copr enable antonpatsev/VictoriaMetrics
+  when: '"antonpatsev/VictoriaMetrics" not in corp_repo.stdout'
+
+- name: Include vmstorage {{ victoriametrics_vmstorage_version }} deploy
+  include: vmstorage.yaml
+  when: inventory_hostname in groups[victoriametrics_vmstorage_group]
+
+- name: Include vminsert {{ victoriametrics_vminsert_version }} deploy
+  include: vminsert.yaml
+  when: inventory_hostname in groups[victoriametrics_vminsert_group]
+
+- name: Include vmselect {{ victoriametrics_vmselect_version }} deploy
+  include: vmselect.yaml
+  when: inventory_hostname in groups[victoriametrics_vmselect_group]
+
+- name: Include vmauth {{ victoriametrics_vmauth_version }} deploy
+  include: vmauth.yaml
+  when: inventory_hostname in groups[victoriametrics_vmauth_group]
+
+- name: Include vmagent {{ victoriametrics_vmagent_version }} deploy
+  include: vmagent.yaml
+  when: inventory_hostname in groups[victoriametrics_vmagent_group]

tasks/vmagent.yaml

@@ -0,0 +1,35 @@
+- name: Install vmagent {{ victoriametrics_vmagent_version }}
+  dnf:
+    name: "vmagent-{{ victoriametrics_vmagent_version }}"
+    state: present
+  register: vmagent_setup
+  notify: Restart vmagent
+
+- name: Template vmagent config
+  template:
+    src: etc/victoriametrics/vmagent/vmagent.conf.j2
+    dest: /etc/victoriametrics/vmagent/vmagent.conf
+  notify: Restart vmagent
+
+- name: Merge prometheus config values
+  set_fact: 
+    victoriametrics_vmagent_prometheus_config_combined: "{{ victoriametrics_vmagent_prometheus_config_default | combine(victoriametrics_vmagent_prometheus_config, recursive=true) | to_nice_yaml(indent=2) }}"
+
+#- name: Template vmagent prometheus config
+#  copy:
+#    content: "{{ victoriametrics_vmagent_prometheus_config }}"
+#    dest: /etc/victoriametrics/vmagent/prometheus.yml
+#  notify: Restart vmagent
+
+- name: Template vmagent prometheus config
+  template:
+    src: etc/victoriametrics/vmagent/prometheus.yml.j2
+    dest: /etc/victoriametrics/vmagent/prometheus.yml
+  notify: Restart vmagent
+
+- name: Enable and start vmagent service
+  systemd:
+    name: vmagent
+    state: started
+    enabled: yes
+    daemon_reload: yes

tasks/vmauth.yaml

@@ -0,0 +1,27 @@
+- name: Install vmauth {{ victoriametrics_vmauth_version }}
+  dnf:
+    name: "vmauth-{{ victoriametrics_vmauth_version }}"
+    state: present
+  register: vmauth_setup
+  notify: Restart vmauth
+
+- name: Template vmauth config
+  template:
+    src: etc/victoriametrics/vmauth/vmauth.conf.j2
+    dest: /etc/victoriametrics/vmauth/vmauth.conf
+  notify: Restart vmauth
+
+- name: Template vmauth users config
+  template:
+    src: etc/victoriametrics/vmauth/config.yml.j2
+    dest: "{{ victoriametrics_vmauth_config_file }}"
+  notify: Restart vmauth
+
+- name: Enable and start vmauth service
+  systemd:
+    name: vmauth
+    state: started
+    enabled: yes
+    daemon_reload: yes
+
+

tasks/vminsert.yaml

@@ -0,0 +1,20 @@
+- name: Install vminsert {{ victoriametrics_vminsert_version }}
+  dnf:
+    name: "vminsert-{{ victoriametrics_vminsert_version }}"
+    state: present
+  register: vminsert_setup
+  notify: Restart vminsert
+
+- name: Template vminsert config
+  template:
+    src: etc/victoriametrics/vmcluster/vminsert.conf.j2
+    dest: /etc/victoriametrics/vmcluster/vminsert.conf
+  notify: Restart vminsert
+
+- name: Enable and start vminsert service
+  systemd:
+    name: vminsert
+    state: started
+    enabled: yes
+    daemon_reload: yes
+

tasks/vmselect.yaml

@@ -0,0 +1,19 @@
+- name: Install vmselect {{ victoriametrics_vmselect_version }}
+  dnf:
+    name: "vmselect-{{ victoriametrics_vmselect_version }}"
+    state: present
+  register: vmselect_setup
+  notify: Restart vmselect
+
+- name: Template vmselect config
+  template:
+    src: etc/victoriametrics/vmcluster/vmselect.conf.j2
+    dest: /etc/victoriametrics/vmcluster/vmselect.conf
+  notify: Restart vmselect
+
+- name: Enable and start vmselect service
+  systemd:
+    name: vmselect
+    state: started
+    enabled: yes
+    daemon_reload: yes

tasks/vmstorage.yaml

@@ -0,0 +1,32 @@
+- name: Install vmstorage {{ victoriametrics_vmstorage_version }}
+  dnf:
+    name: "vmstorage-{{ victoriametrics_vmstorage_version }}"
+    state: present
+  register: vmstorage_setup
+  notify: Restart vmstorage
+
+- name: Create data directory {{ victoriametrics_vmstorage_data_dir }}
+  file:
+    path: "{{ victoriametrics_vmstorage_data_dir }}"
+    state: directory
+    owner: victoriametrics
+    group: victoriametrics
+
+- name: Template vmstorage config
+  template:
+    src: etc/victoriametrics/vmcluster/vmstorage.conf.j2
+    dest: /etc/victoriametrics/vmcluster/vmstorage.conf
+  notify: Restart vmstorage
+
+#- debug:
+#    msg: "{{ groups['vmstorage'] | map('extract', hostvars, ['ansible_host']) | join(':8400,') }}:8400"
+
+#- debug:
+#    msg: "{{ groups['vmstorage'] | map('extract', hostvars, ['inventory_hostname']) | join(':8400,') }}:8400"
+
+- name: Enable and start vmstorage service
+  systemd:
+    name: vmstorage
+    state: started
+    enabled: yes
+    daemon_reload: yes

templates/etc/victoriametrics/vmagent/prometheus.yml.j2

@@ -0,0 +1,50 @@
+# https://docs.victoriametrics.com/
+#
+# https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/deployment/docker/prometheus.yml
+#
+# Example to use vmagent with VictoriaMetrics Cluster 
+#
+# global:
+#   scrape_interval:     1s
+#   evaluation_interval: 1s
+#
+# scrape_configs:
+#   - job_name: 'vmagent'
+#     static_configs:
+#       - targets: ['vmagent:8429']
+#   - job_name: 'vmalert'
+#     static_configs:
+#       - targets: ['vmalert:8880']
+#   - job_name: 'vminsert'
+#     static_configs:
+#       - targets: ['vminsert:8480']
+#   - job_name: 'vmselect'
+#     static_configs:
+#       - targets: ['vmselect:8481']
+#   - job_name: 'vmstorage'
+#     static_configs:
+#       - targets: ['vmstorage:8482']
+#
+# Example to use vmagent with VictoriaMetrics Single 
+{{ victoriametrics_vmagent_prometheus_config_combined }}
+- job_name: 'vmagent'
+  static_configs:
+    - targets: ['127.0.0.1:8429']
+- job_name: 'vminsert'
+  static_configs:
+    - targets:
+{% for insert in groups[victoriametrics_vminsert_group] %}
+      - {{ hostvars[insert]['inventory_hostname'] }}:8480
+{% endfor %}
+- job_name: 'vmselect'
+  static_configs:
+    - targets:
+{% for select in groups[victoriametrics_vmselect_group] %}
+      - {{ hostvars[select]['inventory_hostname'] }}:8481
+{% endfor %}
+- job_name: 'vmstorage'
+  static_configs:
+    - targets:
+{% for storage in groups[victoriametrics_vmstorage_group] %}
+      - {{ hostvars[storage]['inventory_hostname'] }}:8482
+{% endfor %}

templates/etc/victoriametrics/vmagent/vmagent.conf.j2

@@ -0,0 +1,15 @@
+# https://docs.victoriametrics.com/vmagent.html
+#
+# Example command line:
+# /path/to/vmagent -promscrape.config=/path/to/prometheus.yml -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
+#
+# Please note that to write scraped data from vmagent to VictoriaMetrics Cluster you should use url like in this example -remoteWrite.url=http://vminsert-ip:8480/insert/0/prometheus/ .
+# See more information here https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format.
+# 
+# If you only need to collect Influx data, then the following command is sufficient:
+#
+# /path/to/vmagent -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
+#
+# Then send Influx data to http://vmagent-host:8429. See these https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf for more details.
+
+ARGS="-promscrape.config={{ victoriametrics_vmagent_promscrape_config_file }} -remoteWrite.url={% for host in groups[victoriametrics_vmauth_group] %}http://{{ hostvars[host]['inventory_hostname'] }}:8427/api/v1/write{% if not loop.last %},{% endif %}{% endfor %} -remoteWrite.tmpDataPath={{ victoriametrics_vmagent_remoteWrite_tmpDataPath }} -remoteWrite.basicAuth.password=foobar,foobar,foobar -remoteWrite.basicAuth.username=foobar,foobar,foobar"

templates/etc/victoriametrics/vmauth/config.yml.j2

@@ -0,0 +1,92 @@
+# # https://docs.victoriametrics.com/vmauth.html#auth-config
+# # Arbitrary number of usernames may be put here.
+# # Usernames must be unique.
+
+# users:
+#   # Requests with the 'Authorization: Bearer XXXX' header are proxied to http://localhost:8428 .
+#   # For example, http://vmauth:8427/api/v1/query is proxied to http://localhost:8428/api/v1/query
+# - bearer_token: "XXXX"
+#   url_prefix: "http://localhost:8428"
+
+#   # The user for querying local single-node VictoriaMetrics.
+#   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
+#   # will be proxied to http://localhost:8428 .
+#   # For example, http://vmauth:8427/api/v1/query is proxied to http://localhost:8428/api/v1/query
+# - username: "local-single-node"
+#   password: "***"
+#   url_prefix: "http://localhost:8428"
+
+#   # The user for querying local single-node VictoriaMetrics with extra_label team=dev.
+#   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
+#   # will be routed to http://localhost:8428 with extra_label=team=dev query arg.
+#   # For example, http://vmauth:8427/api/v1/query is routed to http://localhost:8428/api/v1/query?extra_label=team=dev
+# - username: "local-single-node"
+#   password: "***"
+#   url_prefix: "http://localhost:8428?extra_label=team=dev"
+
+#   # The user for querying account 123 in VictoriaMetrics cluster
+#   # See https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format
+#   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
+#   # will be load-balanced among http://vmselect1:8481/select/123/prometheus and http://vmselect2:8481/select/123/prometheus
+#   # For example, http://vmauth:8427/api/v1/query is proxied to the following urls in a round-robin manner:
+#   #   - http://vmselect1:8481/select/123/prometheus/api/v1/select
+#   #   - http://vmselect2:8481/select/123/prometheus/api/v1/select
+# - username: "cluster-select-account-123"
+#   password: "***"
+#   url_prefix:
+#   - "http://vmselect1:8481/select/123/prometheus"
+#   - "http://vmselect2:8481/select/123/prometheus"
+
+#   # The user for inserting Prometheus data into VictoriaMetrics cluster under account 42
+#   # See https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format
+#   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
+#   # will be load-balanced between http://vminsert1:8480/insert/42/prometheus and http://vminsert2:8480/insert/42/prometheus
+#   # For example, http://vmauth:8427/api/v1/write is proxied to the following urls in a round-robin manner:
+#   #   - http://vminsert1:8480/insert/42/prometheus/api/v1/write
+#   #   - http://vminsert2:8480/insert/42/prometheus/api/v1/write
+# - username: "cluster-insert-account-42"
+#   password: "***"
+#   url_prefix:
+#   - "http://vminsert1:8480/insert/42/prometheus"
+#   - "http://vminsert2:8480/insert/42/prometheus"
+
+
+#   # A single user for querying and inserting data:
+#   # - Requests to http://vmauth:8427/api/v1/query, http://vmauth:8427/api/v1/query_range
+#   #   and http://vmauth:8427/api/v1/label/<label_name>/values are proxied to the following urls in a round-robin manner:
+#   #     - http://vmselect1:8481/select/42/prometheus
+#   #     - http://vmselect2:8481/select/42/prometheus
+#   #   For example, http://vmauth:8427/api/v1/query is proxied to http://vmselect1:8480/select/42/prometheus/api/v1/query
+#   #   or to http://vmselect2:8480/select/42/prometheus/api/v1/query .
+#   # - Requests to http://vmauth:8427/api/v1/write are proxied to http://vminsert:8480/insert/42/prometheus/api/v1/write
+# - username: "foobar"
+#   url_map:
+#   - src_paths:
+#     - "/api/v1/query"
+#     - "/api/v1/query_range"
+#     - "/api/v1/label/[^/]+/values"
+#     url_prefix:
+#     - "http://vmselect1:8481/select/42/prometheus"
+#     - "http://vmselect2:8481/select/42/prometheus"
+#   - src_paths: ["/api/v1/write"]
+#     url_prefix: "http://vminsert:8480/insert/42/prometheus"
+
+users:
+{% for user in victoriametrics_vmauth_users %}
+  - username: {{ user.username }}
+    password: {{ user.password }}
+    url_map:
+    - src_paths:
+      - "/api/v1/query"
+      - "/api/v1/query_range"
+      - "/api/v1/label/[^/]+/values"
+      url_prefix:
+{% for select in groups[victoriametrics_vmselect_group] %}
+      - "http://{{ hostvars[select]['inventory_hostname'] }}:8481/select/{{ user.id }}/prometheus"
+{% endfor %}
+    - src_paths: ["/api/v1/write"]
+      url_prefix: 
+{% for insert in groups[victoriametrics_vminsert_group] %}
+      - "http://{{ hostvars[insert]['inventory_hostname'] }}:8480/insert/{{ user.id }}/prometheus"
+{% endfor %}
+{% endfor %}

templates/etc/victoriametrics/vmauth/vmauth.conf.j2

@@ -0,0 +1,4 @@
+# https://docs.victoriametrics.com/vmauth.html
+#
+
+ARGS="-auth.config={{ victoriametrics_vmauth_config_file }}"

templates/etc/victoriametrics/vmcluster/vminsert.conf.j2

@@ -0,0 +1,6 @@
+# https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html
+# 
+# https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#architecture-overview
+# 
+#ARGS="-storageNode=127.0.0.1:8400"
+ARGS="-storageNode={{ groups[victoriametrics_vmstorage_group] | map('extract', hostvars, ['inventory_hostname']) | join(':8400,') }}:8400 -replicationFactor={{ victoriametrics_vminsert_replication_factor }}"

templates/etc/victoriametrics/vmcluster/vmselect.conf.j2

@@ -0,0 +1,6 @@
+# https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html
+# 
+# https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#architecture-overview
+# 
+#ARGS="-storageNode=127.0.0.1:8401"
+ARGS="-storageNode={{ groups[victoriametrics_vmselect_group] | map('extract', hostvars, ['inventory_hostname']) | join(':8401,') }}:8401"

templates/etc/victoriametrics/vmcluster/vmstorage.conf.j2

@@ -0,0 +1,5 @@
+# https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html
+# 
+# https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#architecture-overview
+# 
+ARGS="-storageDataPath={{ victoriametrics_vmstorage_data_dir }}"