nginx reverse proxy with noVNC support

K

kristian.kirilov

Well-Known Member
Nov 17, 2016
64
2
48
39
Hello,
I'm trying to configure nginx to act as reverse proxy for my proxmox hosts, everything is great, the noVNC is working, but i cannot upload any file to the datastore.
Config of nginx is attached below:

Code: 
root@proxmox-node-1:~# cat /etc/nginx/nginx.conf
user www-data;
worker_processes auto;
pid /run/nginx.pid;
include /etc/nginx/modules-enabled/*.conf;

events {
        worker_connections 768;
        # multi_accept on;
}

http {

        ##
        # Basic Settings
        ##

        sendfile on;
        tcp_nopush on;
        tcp_nodelay on;
        keepalive_timeout 65;
        types_hash_max_size 2048;
        # server_tokens off;

        # server_names_hash_bucket_size 64;
        # server_name_in_redirect off;

        include /etc/nginx/mime.types;
        default_type application/octet-stream;

        ##
        # SSL Settings
        ##

        ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
        ssl_prefer_server_ciphers on;

        ##
        # Logging Settings
        ##

        access_log /var/log/nginx/access.log;
        error_log /var/log/nginx/error.log;

        ##
        # Gzip Settings
        ##

        gzip on;
        gzip_disable "msie6";

        # gzip_vary on;
        # gzip_proxied any;
        # gzip_comp_level 6;
        # gzip_buffers 16 8k;
        # gzip_http_version 1.1;
        # gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;

        ##
        # Virtual Host Configs
        ##

        include /etc/nginx/conf.d/*.conf;
        include /etc/nginx/sites-enabled/*;
}
root@proxmox-node-1:~# cat /etc/nginx/sites-enabled/proxmox.conf
upstream proxmox {
    server "127.0.0.1";
}

server {
    listen 80 default_server;
    rewrite ^(.*) https://$host$1 permanent;
}

server {
    listen 443;
    server_name 192.168.10.60;

        # Check for cross-framing - nuke bustards
        valid_referers none blocked server_names;
        if ($invalid_referer) {
        return 403;
        }

        # Hint for browsers
        add_header X-Frame-Options SAMEORIGIN;

        # Don't "detect" file type by extension (IE10+?)
        add_header X-Content-Type-Options nosniff;

        # some logs
        access_log /var/log/nginx/proxmox.example.com-ssl-access.log;
        error_log /var/log/nginx/proxmox.example.com-ssl-error.log;

        # load images, backups, iso...
        client_max_body_size 1024m;

        # include default proxy params
        include proxy_params;

    ssl on;
    ssl_certificate /etc/pve/local/pve-ssl.pem;
    ssl_certificate_key /etc/pve/local/pve-ssl.key;
    proxy_redirect off;

    location / {
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection "upgrade";
        proxy_pass https://localhost:8006;
        proxy_buffering off;
        client_max_body_size 0;
        proxy_connect_timeout  3600s;
        proxy_read_timeout  3600s;
        proxy_send_timeout  3600s;
        send_timeout  3600s;
    }

        location ~* ^/(api2|novnc)/ {
                proxy_redirect off;
                proxy_connect_timeout  3600s;
                proxy_read_timeout  3600s;
                proxy_send_timeout  3600s;
                send_timeout  3600s;

                # Magick for VNC
                proxy_set_header Upgrade $http_upgrade;
                proxy_set_header Connection "upgrade";

                include proxy_params;
                proxy_pass https://127.0.0.1:8006;
        }


        location ~* ^/pve2/(?<file>.*)$ {
                gzip_static on;
                root /usr/share/pve-manager;
                try_files /$file @proxmox;
                proxy_connect_timeout  3600s;
                proxy_read_timeout  3600s;
                proxy_send_timeout  3600s;
                send_timeout  3600s;
        }

        # Special for proxmox-5.x
        location ~* ^/proxmox.*\.js$ {
                gzip_static on;
                root /usr/share/javascript/proxmox-widget-toolkit;
                try_files $uri @proxmox;
                proxy_connect_timeout  3600s;
                proxy_read_timeout  3600s;
                proxy_send_timeout  3600s;
                send_timeout  3600s;
        }

        location ~* ^/pve-docs/(?<file>.*)$ {
                gzip_static on;
                root /usr/share/pve-docs;
                try_files /$file @proxmox;
                proxy_connect_timeout  3600s;
                proxy_read_timeout  3600s;
                proxy_send_timeout  3600s;
                send_timeout  3600s;
        }

        location @proxmox {
                # Magick for VNC
                proxy_connect_timeout  3600s;
                proxy_read_timeout  3600s;
                proxy_send_timeout  3600s;
                send_timeout  3600s;
                proxy_set_header Upgrade $http_upgrade;
                proxy_set_header Connection "upgrade";

                include proxy_params;
                proxy_pass https://127.0.0.1:8006;
        }
}
root@proxmox-node-1:~#

This is the error message from nginx log file:

Code: 
2019/03/05 17:09:49 [error] 2590#2590: *9 upstream prematurely closed connection while reading response header from upstream, client: 192.168.0.15, server: 192.168.10.60, request: "POST /api2/json/nodes/proxmox-node-1/storage/local/upload HTTP/1.1", upstream: "https://127.0.0.1:8006/api2/json/nodes/proxmox-node-1/storage/local/upload", host: "192.168.10.60", referrer: "https://192.168.10.60/"

Can somebody help me?
Thanks a lot!
 
The problem is not the size, because if I'm using proxmox web interface on port 8006, I have no problem when uploading the iso.
 
kristian.kirilov said:
The problem is not the size, because if I'm using proxmox web interface on port 8006, I have no problem when uploading the iso.
Click to expand...
Was more if you was hitting any nginx size limits, but the config shows you have this set to 1GB so your test file should be fine.
 
Yeah, any other thoughts about the problem?
I don't know where to look for the problem.. Can you point me to some direction?
 
I checked the syslog, but there is nothing special.
Then I go to /var/log/pve/tasks/, but these logs does not mean anything for me..
I use grep -iR upload and found that:

Code: 
index:UPID:proxmox-node-1:0000392B:000A53ED:5C7020A6:imgcopy::root@pam: 5C7020A7 import failed: /var/tmp/pveupload-038e4e7e03543db7043147c96b06bfac: No such file or directory
index:UPID:proxmox-node-1:00003C8B:000B222A:5C7022B6:imgcopy::root@pam: 5C7022B7 import failed: /var/tmp/pveupload-326bb37d834882925c93656407a0b868: No such file or directory
So I tried to upload the file again, and new file is created in /var/tmp/ when upload is in progress.
Code: 
-rw-------  1 www-data www-data 305135616 Mar  6 15:49 pveupload-cd6a3a146e91d91ce0862e1d4555b3cc

When upload is in 100%, the file disappear, and I log errors like these above.
I don't now if this matter.

Do you know some specific PVE log, which I have to check for this kind of errors?
 
kristian.kirilov said:
Do you know some specific PVE log, which I have to check for this kind of errors?
Click to expand...

I have the same problem. I have a solution just move client_max_body_size 5120m; to top of file, please see my confing file:

Code: 
client_max_body_size 5120m;
upstream proxmox {
    server "bve-01.company.ru";
}
 
server {
    listen 80 default_server;
    rewrite ^(.*) https://$host$1 permanent;
}
 
server {
    listen 443;
    server_name _;
    ssl on;
    ssl_certificate /etc/nginx/fullchain.pem;
    ssl_certificate_key /etc/nginx/privkey.pem;
    proxy_redirect off;
    location / {
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection "upgrade";
        proxy_pass https://localhost:8006;
    proxy_buffering off;
    proxy_connect_timeout  3600s;
        proxy_read_timeout  3600s;
        proxy_send_timeout  3600s;
        send_timeout  3600s;
    }
    location ~* ^/(api2|novnc)/ {
        proxy_redirect off;
        proxy_connect_timeout  3600s;
        proxy_read_timeout  3600s;
        proxy_send_timeout  3600s;
        send_timeout  3600s;

        # Magick for VNC
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection "upgrade";

        include proxy_params;
        proxy_pass https://127.0.0.1:8006;
    }


    location ~* ^/pve2/(?<file>.*)$ {
        gzip_static on;
        root /usr/share/pve-manager;
        try_files /$file @proxmox;
        proxy_connect_timeout  3600s;
        proxy_read_timeout  3600s;
        proxy_send_timeout  3600s;
        send_timeout  3600s;
    }

    # Special for proxmox-5.x
    location ~* ^/proxmox.*\.js$ {
        gzip_static on;
        root /usr/share/javascript/proxmox-widget-toolkit;
        try_files $uri @proxmox;
        proxy_connect_timeout  3600s;
        proxy_read_timeout  3600s;
        proxy_send_timeout  3600s;
        send_timeout  3600s;
    }

    location ~* ^/pve-docs/(?<file>.*)$ {
        gzip_static on;
        root /usr/share/pve-docs;
        try_files /$file @proxmox;
        proxy_connect_timeout  3600s;
        proxy_read_timeout  3600s;
        proxy_send_timeout  3600s;
        send_timeout  3600s;
    }

    location @proxmox {
        # Magick for VNC
        proxy_connect_timeout  3600s;
        proxy_read_timeout  3600s;
        proxy_send_timeout  3600s;
        send_timeout  3600s;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection "upgrade";

        include proxy_params;
        proxy_pass https://127.0.0.1:8006;
    }
}

Will glad if I can help you. After restart nginx it works FINE for upload iso
 
Here is my config with multiple nodes:
Code: 
upstream proxmox {

    server 192.168.1.101:8006;
    server 192.168.1.103:8006;
    server 192.168.1.104:8006;

    # Keep the connection sticky (otherwise the noVNC console redirection will only work randomly).
    ip_hash;
    keepalive 1;
}

server {
    server_name proxmox.mydomain.com;
    listen 80;
    listen [::]:80;
    rewrite ^(.*) https://$host$1 permanent;
    access_log /var/log/nginx/proxmox.access.log;
    error_log /var/log/nginx/proxmox.error.log;
}

server {
    listen 443 ssl http2;
    listen [::]:443 ssl http2;
    server_name proxmox.mydomain.com;
    ssl on;
    ssl_certificate /etc/letsencrypt/live/proxmox.mydomain.com/fullchain.pem; # managed by Certbot
    ssl_certificate_key /etc/letsencrypt/live/proxmox.mydomain.com/privkey.pem; # managed by Certbot

    access_log /var/log/nginx/proxmox.access.log;
    error_log /var/log/nginx/proxmox.error.log;

    proxy_redirect off;
    location / {
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection "upgrade";
        proxy_buffering off;
        client_max_body_size 0;
        proxy_connect_timeout  3600s;
        proxy_read_timeout  3600s;
        proxy_send_timeout  3600s;
        send_timeout  3600s;
        proxy_pass https://proxmox;
    }

}
 
  • Like
Reactions: badsmoke
thanks for the tip with the sticky connection, now the console also runs properly with me
 
You must log in or register to reply here.

About

The Proxmox community has been around for many years and offers help and support for Proxmox VE, Proxmox Backup Server, and Proxmox Mail Gateway.
We think our community is one of the best thanks to people like you!

Quick Navigation

Home Get Subscription Wiki Downloads Proxmox Customer Portal About

Get your subscription!

The Proxmox team works very hard to make sure you are running the best software and getting stable updates and security enhancements, as well as quick enterprise support. Tens of thousands of happy customers have a Proxmox subscription. Get yours easily in our online shop.

Buy now!
Community platform by XenForo® © 2010-2024 XenForo Ltd.
Top Bottom
Back