Andrew Knapp | 22 Jul 00:54 2014
Picon

Hung worker processes after restarting under emperor mode

We are having a problem when we are restarting our app that runs under emperor mode. Sometimes, when we reload the config (an ini file) one or two workers will not die and will start to consume 100% of a cpu, and then die off ~60 seconds later. This will sporadically happen no matter how many workers we spawn.

We are running Django under uwsgi (version 2.0.5) on Ubuntu 14.04 on Amazon EC2.

Configs, logs and strace output (for one of the workers that hung) are below. Has anyone seen/experienced this problem before? My assumption for the 60 second time is the harakiri time, though I'm not 100% sure on that.

Here's the emperor log when a worker was hung:
Mon Jul 21 22:39:41 2014 - [emperor] reload the uwsgi instance <app>
Mon Jul 21 22:40:44 2014 - [emperor] vassal <app> is ready to accept requests

Here's our app ini config (some info removed, though all commands are here that are in the config):
[uwsgi]
uid = <uid>
gid = <gid>
socket = 127.0.0.1:<port>
listen = 16384
workers = 4
threads = 2
thunder-lock = true
max-requests = 20000
harakiri = 60
harakiri-verbose = true
master = true
single-interpreter = true
virtualenv = <virtualenv>
pythonpath = <pythonpath>
env = DJANGO_SETTINGS_MODULE=<module>
module = <wsgi_file>
pidfile2 = <pidfile>
logto2 = <logfile>
logfile-chmod = 644
stats = 127.0.0.1:<stats_port>
post-buffering = 65536
buffer-size = 32768
disable-logging = true
chdir = <dir>

I was able to get an strace off of one of the hung workers, and this is what I got (starting from when they get the signal to reload:
close(4)                                = 0
futex(0x7f3a15c37000, FUTEX_LOCK_PI, 1) = ? ERESTARTNOINTR (To be restarted)
--- SIGHUP {si_signo=SIGHUP, si_code=SI_USER, si_pid=1660, si_uid=601} ---
write(2, "Gracefully killing worker 6 (pid"..., 44) = -1 EPIPE (Broken pipe)
open("/usr/lib/libgcc_s.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 4
fstat(4, {st_mode=S_IFREG|0644, st_size=46184, ...}) = 0
mmap(NULL, 46184, PROT_READ, MAP_PRIVATE, 4, 0) = 0x7f3a15b3d000
close(4)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libgcc_s.so.1", O_RDONLY|O_CLOEXEC) = 4
read(4, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260*\0\0\0\0\0\0"..., 832) = 832
fstat(4, {st_mode=S_IFREG|0644, st_size=90080, ...}) = 0
mmap(NULL, 2185952, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 4, 0) = 0x7f3a09947000
mprotect(0x7f3a0995d000, 2093056, PROT_NONE) = 0
mmap(0x7f3a09b5c000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 4, 0x15000) = 0x7f3a09b5c000
close(4)                                = 0
munmap(0x7f3a15b3d000, 46184)           = 0
tgkill(16665, 16668, SIGRTMIN)          = 0
rt_sigaction(SIGINT, {SIG_DFL, [], SA_RESTORER, 0x7f3a15826340}, {0x460790, [], SA_RESTORER, 0x7f3a15826340}, 8) = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a09907000
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
munmap(0x7f3a098c7000, 262144)          = 0
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3a098c7000
+++ killed by SIGKILL +++

Any help would be appreciated. If anyone wants any other info, just let me know and I'll supply it.

Thanks,
Andy

_______________________________________________
uWSGI mailing list
uWSGI@...
http://lists.unbit.it/cgi-bin/mailman/listinfo/uwsgi
Daniel Nicoletti | 18 Jul 00:58 2014
Picon

cores vs async

With async == 1 the unused async queue is not allocated,
which makes find_first_available_wsgi_req() crash, I was
relying on it to have a predictable way to dispatch requests.

then looking at core/uwsgi.c there is this:

// fill it with default values
for (i = 0; i < uwsgi.async; i++) {
     uwsgi.async_queue_unused[i] = &uwsgi.workers[uwsgi.mywid].cores[i].req;
}

and this:

for (i = 0; i < uwsgi.cores; i++) {
    memset(&uwsgi.workers[uwsgi.mywid].cores[i].req, 0, sizeof(struct
wsgi_request));
    uwsgi.workers[uwsgi.mywid].cores[i].req.async_id = i;
}

If I run --async 10 and --threads 2 then uwsgi.cores is 2
and I wonder why the former loop doesn't crash.

it would be interesting if cores were always async * threads.

best,

--

-- 
Daniel Nicoletti

KDE Developer - http://dantti.wordpress.com
Daniel Nicoletti | 17 Jul 22:43 2014
Picon

connection reset by peer

hi,

I'm running weighttp to measure my app performance
and so far I get lots of:
error: read() failed: Connection reset by peer (104)
The application only writes a reply with
"hello world"
at first I thought it was some OS limit but the
same doesn't happen to nginx.

Any tips on what could be happening?

Thanks

--

-- 
Daniel Nicoletti

KDE Developer - http://dantti.wordpress.com
Jean Finck | 16 Jul 17:11 2014
Picon

- Log encoders + User-defined logvars

I was trying to format the stdout with log encoders and stuck in a doubt.

Is possible to use "user-defined logvars" (http://uwsgi-docs.readthedocs.org/en/latest/LogFormat.html#user-defined-logvars) or something like that in the log encoder formart (http://uwsgi-docs.readthedocs.org/en/latest/LogEncoders.html)?

Most precisely, I would like to append at every log generated by the stdout, which worker/thread made it.

At python:
def application(environ, start_response):
    start_response('200 OK', [('Content-Type','text/html')])
    uwsgi.set_logvar('worker_id', str(uwsgi.worker_id))
    uwsgi.set_logvar('request_id', str(uwsgi.request_id))
    print "Wasabi"
    return ["Hello World"]

At uwsgi.ini:
    log-encoder     = format ${micros} ${worker_id} ${request_id} ${strftime:%%a %%b %%d %%H:%%M:%%S} ${msgnl}


The example above is really simple, but was just to exemplify.. I always can do this "format" at the point the stdout was generated, but as the app already is running and a bit complex, I was trying to find something more modularized and centralized.

There's something that I can do to this approach?
_______________________________________________
uWSGI mailing list
uWSGI@...
http://lists.unbit.it/cgi-bin/mailman/listinfo/uwsgi
}--) - Viktor | 15 Jul 13:08 2014
Picon

http / https setup

Hi,

My site serves both http and https requests. The app decides what to serve over http/https, and redirects accordingly.

When I open the non-ssl site, it works fine, and redirects to the ssl site when needed, but the ssl site does not show up, even the nginx and uwsgi logs are mute after the redirect.

Do I need to run separate app instances on different ports?
Do I need to set up https certificates in the ini file too?

I run uwsgi through a socket. My ini file is:
[uwsgi]
socket = 127.0.0.1:8080
chdir = /home/appserver/example
home = /home/appserver/.virtualenvs/example
env = DJANGO_SETTINGS_MODULE=example.settings
module = example.wsgi:application
processes = 4
threads = 2
memory-report = true
stats = 127.0.0.1:9191
lazy-apps = true
master-fifo = /tmp/example.fifo
log-master = true
logto = /home/appserver/logs/uwsgi.log

The nginx config is:
server {
    listen 80 default_server;
    server_name my.example.com;

    root /home/appserver/example/static;
    index index.html index.htm;

    location / {
        include            uwsgi_params;
        uwsgi_pass         127.0.0.1:8080;

        proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
        proxy_redirect off;
        proxy_set_header   X-Forwarded-Host $server_name;
        proxy_set_header        Host            $host;
        proxy_set_header        X-Real-IP       $remote_addr;
        proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_pass_header Server;
        proxy_set_header X-Scheme $scheme;
    }
}

server {
    listen 443;
    server_name my.example.com;
    ssl                  on;
    ssl_certificate      /path/to/example.crt;
    ssl_certificate_key  /path/to/example.key;
    # no security problem here, since / is alway passed to upstream
    root /home/appserver/example/static;
    location / {
        include            uwsgi_params;
        uwsgi_pass         127.0.0.1:8080;

        proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
        proxy_redirect off;
        proxy_set_header   X-Forwarded-Host $server_name;
        proxy_set_header        Host            $host;
        proxy_set_header        X-Real-IP       $remote_addr;
        proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
       proxy_pass_header Server;
        proxy_set_header X-Scheme $scheme;
       proxy_ignore_client_abort on;
    }
}
_______________________________________________
uWSGI mailing list
uWSGI@...
http://lists.unbit.it/cgi-bin/mailman/listinfo/uwsgi
Roberto De Ioris | 12 Jul 11:52 2014
Picon

subscription mountpoints in uWSGI 2.1

Hi all,

in the 2.1 branch a lot of new features and improvements to the
subscription system have been added.

The most interesting is probably mountpoints:

http://uwsgi-docs.readthedocs.org/en/latest/SubscriptionServer.html#mountpoints-uwsgi-2-1

In addition to this, backup nodes can now be configured (just add backup=n
to subscribe2, and it will be used only when all of the non-backup nodes
are death [the n is used to give precedence to backup nodes])

The http router can now speak http to backends (in addition to uwsgi) If
you have an instance (like a node or a tomcat server) you want to directly
subscribe to the httprouter just run it as:

[uwsgi]
subscribe2 = server=/run/ss,key=mydomain,addr=127.0.0.1:8000,proto=http
; bind node to port 8000
attach-daemon = node mydaemon.js 127.0.0.1:8000

Finally, load balancing algorithms are now pluggable (you can define new
algorithms in plugins), and you can choose which one to use for each
subscription key

--subscribe2 ...,algo=iphash

will use the new iphash algorithm instead of the wrr (weighted round
robin) one

There are currently 4 embedded algos:

wrr -> weighted round robin with backup support
lrc -> least reference counting with backup
wlrc -> weighted least reference counting with backup
iphash -> sticky connect to the same node using the hash of the source ip

Every report will be very useful

Regards

--

-- 
Roberto De Ioris
http://unbit.it
Daniel Nicoletti | 11 Jul 16:25 2014
Picon

option typo

Hi,

I was trying to figure out if the
relation threads - uwsgi_socket was 1:1
and found this "sacket" typo:

core/uwsgi.c:   {"shared-socket", required_argument, 0, "create a
shared sacket for advanced jailing or ipc",
uwsgi_opt_add_shared_socket, NULL, 0},

core/uwsgi.c:   {"undeferred-shared-socket", required_argument, 0,
"create a shared sacket for advanced jailing or ipc (undeferred
mode)", uwsgi_opt_add_shared_socket, NULL, 0},

--

-- 
Daniel Nicoletti

KDE Developer - http://dantti.wordpress.com
Daniel Nicoletti | 10 Jul 18:34 2014
Picon

Qt event loop

Hi,
I'm trying to include the Qt event loop you wrote,
but I get a crash as soon as:

 int ret = uwsgi.wait_read_hook(wsgi_req->fd, uwsgi.socket_timeout);

is called.

When I attach gdb to it the process get stuck and
I can't get a decent backtrace, do you have an idea
of what can be wrong or how to get a proper bt?

Best,

--

-- 
Daniel Nicoletti

KDE Developer - http://dantti.wordpress.com
Jean Finck | 7 Jul 14:31 2014
Picon

thunder-lock + max-requests = deadlock warning

I would like to know if is safe to use "max-requests" while "thunder-lock" is enabled.
Since I started to use "max-requests" I noticed some new messages in the log, saying:


Fri Jul 06 09:25:26 - [deadlock-detector] a process holding a robust mutex died. recovering...
Fri Jul 06 09:25:26 - ...The work of process 12173 is done. Seeya!
Fri Jul 06 09:25:27 - worker 2 killed successfully (pid: 12173)
Fri Jul 06 09:25:27 - Respawned uWSGI worker 2 (new pid: 12252)


But I have noticed too, that is inconsistent. What I mean is, every time a process hits five requests, the process restarts, but the message of deadlock appear only some times.



[uwsgi]
master       = true
die-on-term
uid          = nginx
gid          = nginx
pidfile      = /var/run/uwsgi.pid
socket       = /tmp/uwsgi.sock
stats        = /tmp/uwsgi-stats.sock
chown-socket = nginx:nginx

daemonize       = /var/log/uwsgi/n.log
log-maxsize     = 104857600
log-date        = %%a %%b %%d %%H:%%M:%%S
log-5xx
log-zero
log-ioerror     = true
log-slow        = 5000
log-big         = 2048
disable-logging

chdir  = ...
module = wsgi

cheaper-algo    = spare
cheaper         = 2
cheaper-initial = 2
cheaper-step    = 1

cpu-affinity  = 1
workers       = 4
threads       = 5
vacuum        = true
max-requests  = 5 (tests purposes)
#enable-threads
thunder-lock

_______________________________________________
uWSGI mailing list
uWSGI@...
http://lists.unbit.it/cgi-bin/mailman/listinfo/uwsgi
alonn | 5 Jul 11:42 2014
Picon

Running a wsgi app with uwsgi - how many times is the wsgi.py file called

Let's say I'm configuring uwsgi to run with 4 processes 
And lets say I have some startup code (that should run once - and only once) loading data to memory in my wsgi.py file as suggested here

Would the wsgi.py code would be run everytime a worker process is started (and restarted)? 

I heard a mention about uwsgi "master" process - should I use one and how?
Are there specific configuration options I should use for this?

I'm using uwsgi 2.0.4+,django behind nginx (with unix socket) 

I'll be glad for help with this subject

Tel:972-54-6734469
_______________________________________________
uWSGI mailing list
uWSGI@...
http://lists.unbit.it/cgi-bin/mailman/listinfo/uwsgi
alonn | 5 Jul 10:46 2014
Picon

Schooling me about Uwsgi workers, processes and shared memory space (using uwsgi with djagno and behind nginx with unix socket)

After reading lots and lots of Uwsgi docs I Still don't think I understand exactly what memory space is shared between uwsgi workers running the web server. I have some memory based shared config (and configurable from the outside) everything running Django should have access to.
Are uswgi worker processes a **real** process? if so what memory space is shared between them? 

Yes, I know I can use redis/memcache etc for inter process communication but it's both ugly and slower then "in memory".

Thanks!


Tech blog: https://medium.com/ <at> alonisser and the hebrew one: 4p-tech.co.il/blog
Tel:972-54-6734469
_______________________________________________
uWSGI mailing list
uWSGI@...
http://lists.unbit.it/cgi-bin/mailman/listinfo/uwsgi

Gmane