2025-07-01T03:50:40.054Z ERROR [MetricsCollector] Uncaught exception in Periodical
org.graylog.shaded.opensearch2.org.opensearch.OpenSearchStatusException: OpenSearch exception [type=index_not_found_exception, reason=no such index [gl-datanode-metrics]]
        at org.graylog.shaded.opensearch2.org.opensearch.rest.BytesRestResponse.errorFromXContent(BytesRestResponse.java:210) ~[graylog-storage-opensearch2-6.1.12.jar:?]
        at org.graylog.shaded.opensearch2.org.opensearch.client.RestHighLevelClient.parseEntity(RestHighLevelClient.java:2235) ~[graylog-storage-opensearch2-6.1.12.jar:?]
        at org.graylog.shaded.opensearch2.org.opensearch.client.RestHighLevelClient.parseResponseException(RestHighLevelClient.java:2212) ~[graylog-storage-opensearch2-6.1.12.jar:?]
        at org.graylog.shaded.opensearch2.org.opensearch.client.RestHighLevelClient.internalPerformRequest(RestHighLevelClient.java:1931) ~[graylog-storage-opensearch2-6.1.12.jar:?]
        at org.graylog.shaded.opensearch2.org.opensearch.client.RestHighLevelClient.performRequest(RestHighLevelClient.java:1884) ~[graylog-storage-opensearch2-6.1.12.jar:?]
        at org.graylog.shaded.opensearch2.org.opensearch.client.RestHighLevelClient.performRequestAndParseEntity(RestHighLevelClient.java:1852) ~[graylog-storage-opensearch2-6.1.12.jar:?]
        at org.graylog.shaded.opensearch2.org.opensearch.client.RestHighLevelClient.search(RestHighLevelClient.java:1095) ~[graylog-storage-opensearch2-6.1.12.jar:?]
        at org.graylog.datanode.periodicals.MetricsCollector.getPreviousMetricsForCluster(MetricsCollector.java:172) ~[graylog-datanode.jar:?]
        at org.graylog.datanode.periodicals.MetricsCollector.lambda\$doRun\$0(MetricsCollector.java:123) ~[graylog-datanode.jar:?]
        at java.base/java.util.Optional.ifPresent(Unknown Source) ~[?:?]
        at org.graylog.datanode.periodicals.MetricsCollector.doRun(MetricsCollector.java:109) ~[graylog-datanode.jar:?]
        at org.graylog2.plugin.periodical.Periodical.run(Periodical.java:99) [graylog2-server-6.1.12.jar:?]
        at java.base/java.util.concurrent.Executors\$RunnableAdapter.call(Unknown Source) [?:?]
        at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) [?:?]
        at java.base/java.util.concurrent.ScheduledThreadPoolExecutor\$ScheduledFutureTask.run(Unknown Source) [?:?]
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) [?:?]
        at java.base/java.util.concurrent.ThreadPoolExecutor\$Worker.run(Unknown Source) [?:?]
        at java.base/java.lang.Thread.run(Unknown Source) [?:?]
        Suppressed: org.graylog.shaded.opensearch2.org.opensearch.client.ResponseException: method [POST], host [https://datanode:9200], URI [/gl-datanode-metrics/_search?typed_keys=true&max_concurrent_shard_requests=5&ignore_unavailable=false&expand_wildcards=open&allow_no_indices=true&ignore_throttled=true&search_type=query_then_fetch&batched_reduce_size=512&ccs_minimize_roundtrips=true], status line [HTTP/1.1 404 Not Found]
{"error":{"root_cause":[{"type":"index_not_found_exception","reason":"no such index [gl-datanode-metrics]","index":"gl-datanode-metrics","resource.id":"gl-datanode-metrics","resource.type":"index_or_alias","index_uuid":"_na_"}],"type":"index_not_found_exception","reason":"no such index [gl-datanode-metrics]","index":"gl-datanode-metrics","resource.id":"gl-datanode-metrics","resource.type":"index_or_alias","index_uuid":"_na_"},"status":404}
                at org.graylog.shaded.opensearch2.org.opensearch.client.RestClient.convertResponse(RestClient.java:376) ~[graylog-storage-opensearch2-6.1.12.jar:?]
                at org.graylog.shaded.opensearch2.org.opensearch.client.RestClient.performRequest(RestClient.java:346) ~[graylog-storage-opensearch2-6.1.12.jar:?]
                at org.graylog.shaded.opensearch2.org.opensearch.client.RestClient.performRequest(RestClient.java:321) ~[graylog-storage-opensearch2-6.1.12.jar:?]
                at org.graylog.shaded.opensearch2.org.opensearch.client.RestHighLevelClient.internalPerformRequest(RestHighLevelClient.java:1918) ~[graylog-storage-opensearch2-6.1.12.jar:?]
                at org.graylog.shaded.opensearch2.org.opensearch.client.RestHighLevelClient.performRequest(RestHighLevelClient.java:1884) ~[graylog-storage-opensearch2-6.1.12.jar:?]
                at org.graylog.shaded.opensearch2.org.opensearch.client.RestHighLevelClient.performRequestAndParseEntity(RestHighLevelClient.java:1852) ~[graylog-storage-opensearch2-6.1.12.jar:?]
                at org.graylog.shaded.opensearch2.org.opensearch.client.RestHighLevelClient.search(RestHighLevelClient.java:1095) ~[graylog-storage-opensearch2-6.1.12.jar:?]
                at org.graylog.datanode.periodicals.MetricsCollector.getPreviousMetricsForCluster(MetricsCollector.java:172) ~[graylog-datanode.jar:?]
                at org.graylog.datanode.periodicals.MetricsCollector.lambda\$doRun\$0(MetricsCollector.java:123) ~[graylog-datanode.jar:?]
                at java.base/java.util.Optional.ifPresent(Unknown Source) ~[?:?]
                at org.graylog.datanode.periodicals.MetricsCollector.doRun(MetricsCollector.java:109) ~[graylog-datanode.jar:?]
                at org.graylog2.plugin.periodical.Periodical.run(Periodical.java:99) [graylog2-server-6.1.12.jar:?]
                at java.base/java.util.concurrent.Executors\$RunnableAdapter.call(Unknown Source) [?:?]
                at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) [?:?]
                at java.base/java.util.concurrent.ScheduledThreadPoolExecutor\$ScheduledFutureTask.run(Unknown Source) [?:?]
                at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) [?:?]
                at java.base/java.util.concurrent.ThreadPoolExecutor\$Worker.run(Unknown Source) [?:?]
                at java.base/java.lang.Thread.run(Unknown Source) [?:?]
















YAML File:
# For DataNode setup, graylog starts with a preflight UI, this is a change from just using OpenSearch/Elasticsearch.
# Please take a look at the README at the top of this repo or the regular docs for more info.

services:
  # MongoDB: https://hub.docker.com/_/mongo/
  mongodb:
    image: "mongo:6.0"
    restart: "on-failure"
    networks:
      - graylog
    volumes:
      - "mongodb_data:/data/db"
      - "mongodb_config:/data/configdb"

  # For DataNode setup, graylog starts with a preflight UI, this is a change from just using OpenSearch/Elasticsearch.
  # Please take a look at the README at the top of this repo or the regular docs for more info.
  # Graylog Data Node: https://hub.docker.com/r/graylog/graylog-datanode
  datanode:
    image: "${DATANODE_IMAGE:-graylog/graylog-datanode:6.1}"
    hostname: "datanode"
    environment:
      GRAYLOG_DATANODE_NODE_ID_FILE: "/var/lib/graylog-datanode/node-id"
      # GRAYLOG_DATANODE_PASSWORD_SECRET and GRAYLOG_PASSWORD_SECRET MUST be the same value
      GRAYLOG_DATANODE_PASSWORD_SECRET: "${GRAYLOG_PASSWORD_SECRET:?Please configure GRAYLOG_PASSWORD_SECRET in the .env file}"
      GRAYLOG_DATANODE_MONGODB_URI: "mongodb://mongodb:27017/graylog"
    ulimits:
      memlock:
        hard: -1
        soft: -1
      nofile:
        soft: 65536
        hard: 65536
    ports:
      - "8999:8999/tcp"   # DataNode API
      - "9200:9200/tcp"
      - "9300:9300/tcp"
    networks:
      - graylog
    volumes:
      - "graylog-datanode:/var/lib/graylog-datanode"
    restart: "on-failure"

  # Graylog: https://hub.docker.com/r/graylog/graylog-enterprise
  graylog:
    hostname: "server"
    image: "${GRAYLOG_IMAGE:-graylog/graylog:6.1}"
    depends_on:
      mongodb:
        condition: "service_started"
      datanode:
        condition: "service_started"
    entrypoint: "/usr/bin/tini --  /docker-entrypoint.sh"
    environment:
      GRAYLOG_NODE_ID_FILE: "/usr/share/graylog/data/data/node-id"
      # GRAYLOG_DATANODE_PASSWORD_SECRET and GRAYLOG_PASSWORD_SECRET MUST be the same value
      GRAYLOG_PASSWORD_SECRET: "${GRAYLOG_PASSWORD_SECRET:?Please configure GRAYLOG_PASSWORD_SECRET in the .env file}"
      GRAYLOG_ROOT_PASSWORD_SHA2: "${GRAYLOG_ROOT_PASSWORD_SHA2:?Please configure GRAYLOG_ROOT_PASSWORD_SHA2 in the .env file}"
      GRAYLOG_HTTP_BIND_ADDRESS: "0.0.0.0:9000"
      GRAYLOG_HTTP_EXTERNAL_URI: "http://localhost:9000/"
      GRAYLOG_MONGODB_URI: "mongodb://mongodb:27017/graylog"
    ports:
    - "5044:5044/tcp"   # Beats
    - "5140:5140/udp"   # Syslog
    - "5140:5140/tcp"   # Syslog
    - "5555:5555/tcp"   # RAW TCP
    - "5555:5555/udp"   # RAW UDP
    - "9000:9000/tcp"   # Server API
    - "12201:12201/tcp" # GELF TCP
    - "12201:12201/udp" # GELF UDP
    #- "10000:10000/tcp" # Custom TCP port
    #- "10000:10000/udp" # Custom UDP port
    - "13301:13301/tcp" # Forwarder data
    - "13302:13302/tcp" # Forwarder config
    networks:
      - graylog
    volumes:
      - "graylog_data:/usr/share/graylog/data/data"
    restart: "on-failure"

networks:
  graylog:
    driver: "bridge"

volumes:
  mongodb_data:
  mongodb_config:
  graylog-datanode:
  graylog_data: