Loki Behind reverse proxy 404

Grafana Loki
, , , ,
1

Hi there,

I’m facing a really annoying error when setting up my grafana stack server.
I have a nginx reverse proxying for multiple applications that I’ve setted up. here’s my proxy.conf:
upstream grafana {
server community.grafana.com;
}

upstream loki {
server localhost:3100;
}

upstream prometheus {
server localhost:9090;
}

upstream blackbox {
server localhost:9115;
}

upstream node {
server localhost:9100;
}

#Redirects HTTP to HTTPS
server {
listen 80;
listen [::]:80;
server_name <mydomainname.provider>;

location ~* ^\/grafana\/loki\/*[.+]* {
    proxy_http_version 1.1;
    proxy_set_header Host $http_host;
    proxy_pass http://loki;
    proxy_set_header Upgrade $http_upgrade;
    proxy_set_header Connection $connection_upgrade;
    proxy_set_header Connection "Keep-Alive";
    proxy_set_header Proxy-Connection "Keep-Alive";
    proxy_redirect off;
}

location / {
return 301 https://$server_name$request_uri;
}

}

#HTTPS Server
server {
listen 443 ssl http2;
listen [::]:443 ssl http2;
server_name <mydomainname.provider>;
ssl_certificate /etc/ssl/certs/mydomainname.pem;
ssl_certificate_key /etc/ssl/certs/mydomainname.pem;
ssl_client_certificate /etc/ssl/certs/mydomainname.pem;
ssl_protocols TLSv1.1 TLSv1.2 TLSv1.3;
ssl_verify_client optional;

location /prometheus/blackbox/ {
    proxy_set_header Host $http_host;
    proxy_pass http://blackbox;
#     include proxy_params;
}

location /prometheus/node/ {
    proxy_set_header Host $http_host;
    proxy_pass http://node;
#     include proxy_params;
}

location /prometheus {
    proxy_set_header Host $http_host;
    proxy_pass http://prometheus;
}

location ~* ^\/grafana\/loki\/*[.+]* {
    proxy_http_version 1.1;
    proxy_set_header Host $http_host;
    proxy_pass http://loki;
    proxy_set_header Upgrade $http_upgrade;
    proxy_set_header Connection $connection_upgrade;
    proxy_set_header Connection "Keep-Alive";
    proxy_set_header Proxy-Connection "Keep-Alive";
    proxy_redirect off;
}

location /grafana/api/live/ {
    proxy_http_version 1.1;
    proxy_set_header Upgrade $http_upgrade;
    proxy_set_header Connection $connection_upgrade;
    proxy_set_header Host $http_host;
    proxy_pass http://grafana;
}

location /grafana {
    proxy_set_header Host $http_host;
    proxy_pass http://grafana;
}


location / {
    return 301 /grafana/;
}

}

map $http_upgrade $connection_upgrade {
default upgrade;
‘’ close;
}

When I check server logs for errors, no apparent error shows up, although, when trying to add the loki datasource to grafana, I’m facing a 404 error:

image
image760×482 27.3 KB

I’ve tried a different proxy settings but none worked for me. I’m able to access all loki APIs and even my promtail is being able to push logs to it, returning 204.

Here’s my loki server conf:
auth_enabled: false

server:
http_listen_network: tcp
http_listen_address: localhost
http_listen_port: 3100
http_path_prefix: /grafana/loki
grpc_listen_port: 9096

common:
instance_addr: 127.0.0.1
path_prefix: /tmp/loki
storage:
filesystem:
chunks_directory: /tmp/loki/chunks
rules_directory: /tmp/loki/rules
replication_factor: 1
ring:
kvstore:
store: inmemory

query_range:
results_cache:
cache:
embedded_cache:
enabled: true
max_size_mb: 100

schema_config:
configs:
- from: 2020-10-24
store: boltdb-shipper
object_store: filesystem
schema: v11
index:
prefix: index_
period: 24h

analytics:
reporting_enabled: false

Here’s my promtail conf:
server:
http_listen_port: 9080
grpc_listen_port: 0

positions:
filename: /etc/grafana/promtail/temp/positions.yaml

clients:

scrape_configs:

  • job_name: grafana
    static_configs:

    • targets:
      • localhost
        labels:
        job: nginx
        path: /var/log/nginx/grafana/*log

  • job_name: nginx
    static_configs:

    • targets:
      • localhost
        labels:
        job: nginx
        path: /var/log/nginx/*log

  • job_name: system
    static_configs:

    • targets:
      • localhost
        labels:
        job: varlogs
        path: /var/log/*log

I don’t know exaclty what I’m messing up, everything looks working, but when trying to add to grafana, I’m getting a 404. Anyone can help?

Nginx version: 1.24
Loki version: 2.8.x
Promtail version: 2.8.2
Grafana version: 10.0.0

Edit: I’m NOT running on docker