Hi,
I’m setting up a media server, i’ve tried with some different tune-up but i’ve got a problem that i cannot solve, when i have just a few users like 10\20 all goes fine, but when i start rising users, up to something like 50\60, the stream stops, just go black, and after some time, it starts back.
on htop the cpu and memory are almost fully free, on iftop i can see that all the interfaces goes from 70\80 Mbps to 10\20 Kbps. And the jconsole disconnects from the server, even if i’m connected locally.
The server have
2x Intel® Xeon® CPU E5-2603 0 @ 1.80GHz
32Gb ram
networking:
2x 10Gb solar flare
4x 1Gb intel
and the storage is not the problem.
Java :
java version “1.6.0_37”
Java™ SE Runtime Environment (build 1.6.0_37-b06)
Java HotSpot™ 64-Bit Server VM (build 20.12-b01, mixed mode)
The operating system is Gentoo with hardened kernel and some tweeking :
Meminfo
MemTotal: 32921128 kB
MemFree: 24351228 kB
Buffers: 94424 kB
Cached: 1821916 kB
SwapCached: 0 kB
Active: 725092 kB
Inactive: 1202088 kB
Active(anon): 10988 kB
Inactive(anon): 124 kB
Active(file): 714104 kB
Inactive(file): 1201964 kB
Unevictable: 0 kB
Mlocked: 0 kB
SwapTotal: 4194300 kB
SwapFree: 4194300 kB
Dirty: 0 kB
Writeback: 0 kB
AnonPages: 10864 kB
Mapped: 8312 kB
Shmem: 256 kB
Slab: 93916 kB
SReclaimable: 54860 kB
SUnreclaim: 39056 kB
KernelStack: 872 kB
PageTables: 1756 kB
NFS_Unstable: 0 kB
Bounce: 0 kB
WritebackTmp: 0 kB
CommitLimit: 17509136 kB
Committed_AS: 50092 kB
VmallocTotal: 34359738367 kB
VmallocUsed: 370960 kB
VmallocChunk: 34342325304 kB
AnonHugePages: 0 kB
HugePages_Total: 3072
HugePages_Free: 3072
HugePages_Rsvd: 0
HugePages_Surp: 0
Hugepagesize: 2048 kB
DirectMap4k: 2532 kB
DirectMap2M: 18446744073709549568 kB
DirectMap1G: 31457280 kB
Sysctl
# /etc/sysctl.conf
#
# For more information on how this file works, please see
# the manpages sysctl(8) and sysctl.conf(5).
#
# In order for this file to work properly, you must first
# enable 'Sysctl support' in the kernel.
#
# Look in /proc/sys/ for all the things you can setup.
#
# Disables packet forwarding
net.ipv4.ip_forward = 0
# Disables IP dynaddr
#net.ipv4.ip_dynaddr = 0
# Disable ECN
#net.ipv4.tcp_ecn = 0
# Enables source route verification
net.ipv4.conf.default.rp_filter = N
# Enable reverse path
net.ipv4.conf.all.rp_filter = 1
# Enable SYN cookies (yum!)
# http://cr.yp.to/syncookies.html
net.ipv4.tcp_syncookies = 0
# Disable source route
#net.ipv4.conf.all.accept_source_route = 0
#net.ipv4.conf.default.accept_source_route = 0
# Disable redirects
#net.ipv4.conf.all.accept_redirects = 0
#net.ipv4.conf.default.accept_redirects = 0
# Disable secure redirects
#net.ipv4.conf.all.secure_redirects = 0
#net.ipv4.conf.default.secure_redirects = 0
# Ignore ICMP broadcasts
#net.ipv4.icmp_echo_ignore_broadcasts = 1
# Disables the magic-sysrq key
#kernel.sysrq = 0
# When the kernel panics, automatically reboot in 3 seconds
#kernel.panic = 3
# Allow for more PIDs (cool factor!); may break some programs
#kernel.pid_max = 999999
# You should compile nfsd into the kernel or add it
# to modules.autoload for this to work properly
# TCP Port for lock manager
#fs.nfs.nlm_tcpport = 0
# UDP Port for lock manager
#fs.nfs.nlm_udpport = 0
### custom swap
vm.swappiness = 0
## custom network
# increase TCP max buffer size setable using setsockopt()
net.core.rmem_max = 67108864
net.core.wmem_max = 67108864
# increase Linux autotuning TCP buffer limit
net.ipv4.tcp_rmem = 4096 87380 67108864
net.ipv4.tcp_wmem = 4096 65536 67108864
# increase the length of the processor input queue
net.core.netdev_max_backlog = 250000
net.ipv4.tcp_congestion_control=cubic
## custom fs
fs.inotify.max_user_watches = 65536
fs.file-max = 65535
## Huge Pages
vm.nr_hugepages = 3072
The wowza config are the following :
Setenv.sh
#!/bin/sh
_EXECJAVA=java
JAVA_OPTS="-d64 -Xmx8000M -XX:MaxPermSize=4000M -Xms8000M -Xss256k -XX:+AlwaysPreTouch -XX:ParallelCMSThreads=8 -XX:ParallelGCThreads=8"
JAVA_OPTS="$JAVA_OPTS -server"
JAVA_OPTS="$JAVA_OPTS -XX:+UseConcMarkSweepGC -XX:+UseLargePages -XX:+UseParNewGC -XX:NewSize=512m"
JAVA_OPTS="$JAVA_OPTS -Djava.net.preferIPv4Stack=true"
WMSAPP_HOME=/usr/local/WowzaMediaServer
WMSCONFIG_HOME=/usr/local/WowzaMediaServer
WMSCONFIG_URL=
export WMSAPP_HOME WMSCONFIG_HOME JAVA_OPTS _EXECJAVA
VHost.xml
<Root>
<VHost>
<HostPortList>
<HostPort>
<ProcessorCount>16</ProcessorCount>
<IpAddress>*</IpAddress>
<Port>1935</Port>
<SocketConfiguration>
<ReuseAddress>false</ReuseAddress>
<ReceiveBufferSize>131072</ReceiveBufferSize>
<ReadBufferSize>131072</ReadBufferSize>
<SendBufferSize>131072</SendBufferSize>
<KeepAlive>true</KeepAlive>
<AcceptorBackLog>100</AcceptorBackLog>
</SocketConfiguration>
<HTTPStreamerAdapterIDs>cupertinostreaming,smoothstreaming,sanjosestreaming,dvrchunkstreaming</HTTPStreamerAdapterIDs>
<HTTPProviders>
<HTTPProvider>
<BaseClass>com.wowza.wms.http.HTTPCrossdomain</BaseClass>
<RequestFilters>*crossdomain.xml</RequestFilters>
<AuthenticationMethod>none</AuthenticationMethod>
</HTTPProvider>
<HTTPProvider>
<BaseClass>com.wowza.wms.http.HTTPClientAccessPolicy</BaseClass>
<RequestFilters>*clientaccesspolicy.xml</RequestFilters>
<AuthenticationMethod>none</AuthenticationMethod>
</HTTPProvider>
<HTTPProvider>
<BaseClass>com.wowza.wms.http.HTTPProviderMediaList</BaseClass>
<RequestFilters>*jwplayer.rss|*jwplayer.smil|*medialist.smil|*manifest-rtmp.f4m</RequestFilters>
<AuthenticationMethod>none</AuthenticationMethod>
</HTTPProvider>
<HTTPProvider>
<BaseClass>com.wowza.wms.http.HTTPServerVersion</BaseClass>
<RequestFilters>*</RequestFilters>
<AuthenticationMethod>none</AuthenticationMethod>
</HTTPProvider>
</HTTPProviders>
</HostPort>
<HostPort>
<ProcessorCount>16</ProcessorCount>
<IpAddress>*</IpAddress>
<Port>8086</Port>
<SocketConfiguration>
<ReuseAddress>false</ReuseAddress>
<ReceiveBufferSize>131072</ReceiveBufferSize>
<ReadBufferSize>131072</ReadBufferSize>
<SendBufferSize>131072</SendBufferSize>
<KeepAlive>true</KeepAlive>
<AcceptorBackLog>100</AcceptorBackLog>
</SocketConfiguration>
<HTTPStreamerAdapterIDs></HTTPStreamerAdapterIDs>
<HTTPProviders>
<HTTPProvider>
<BaseClass>com.wowza.wms.http.streammanager.HTTPStreamManager</BaseClass>
<RequestFilters>streammanager*</RequestFilters>
<AuthenticationMethod>admin-digest</AuthenticationMethod>
</HTTPProvider>
<HTTPProvider>
<BaseClass>com.wowza.wms.http.HTTPServerInfoXML</BaseClass>
<RequestFilters>serverinfo*</RequestFilters>
<AuthenticationMethod>admin-digest</AuthenticationMethod>
</HTTPProvider>
<HTTPProvider>
<BaseClass>com.wowza.wms.http.HTTPConnectionInfo</BaseClass>
<RequestFilters>connectioninfo*</RequestFilters>
<AuthenticationMethod>admin-digest</AuthenticationMethod>
</HTTPProvider>
<HTTPProvider>
<BaseClass>com.wowza.wms.http.HTTPConnectionCountsXML</BaseClass>
<RequestFilters>connectioncounts*</RequestFilters>
<AuthenticationMethod>admin-digest</AuthenticationMethod>
</HTTPProvider>
<HTTPProvider>
<BaseClass>com.wowza.wms.transcoder.httpprovider.HTTPTranscoderThumbnail</BaseClass>
<RequestFilters>transcoderthumbnail*</RequestFilters>
<AuthenticationMethod>admin-digest</AuthenticationMethod>
</HTTPProvider>
<HTTPProvider>
<BaseClass>com.wowza.wms.http.HTTPProviderMediaList</BaseClass>
<RequestFilters>medialist*</RequestFilters>
<AuthenticationMethod>admin-digest</AuthenticationMethod>
</HTTPProvider>
<HTTPProvider>
<BaseClass>com.wowza.wms.livestreamrecord.http.HTTPLiveStreamRecord</BaseClass>
<RequestFilters>livestreamrecord*</RequestFilters>
<AuthenticationMethod>admin-digest</AuthenticationMethod>
</HTTPProvider>
<HTTPProvider>
<BaseClass>com.wowza.wms.http.HTTPServerVersion</BaseClass>
<RequestFilters>*</RequestFilters>
<AuthenticationMethod>none</AuthenticationMethod>
</HTTPProvider>
</HTTPProviders>
</HostPort>
</HostPortList>
<HTTPStreamerAdapters>
<HTTPStreamerAdapter>
<ID>smoothstreaming</ID>
<Name>smoothstreaming</Name>
<Properties>
</Properties>
</HTTPStreamerAdapter>
<HTTPStreamerAdapter>
<ID>cupertinostreaming</ID>
<Name>cupertinostreaming</Name>
<Properties>
</Properties>
</HTTPStreamerAdapter>
<HTTPStreamerAdapter>
<ID>sanjosestreaming</ID>
<Name>sanjosestreaming</Name>
<Properties>
</Properties>
</HTTPStreamerAdapter>
<HTTPStreamerAdapter>
<ID>dvrchunkstreaming</ID>
<Name>dvrchunkstreaming</Name>
<Properties>
</Properties>
</HTTPStreamerAdapter>
</HTTPStreamerAdapters>
<HandlerThreadPool>
<PoolSize>1200</PoolSize>
</HandlerThreadPool>
<TransportThreadPool>
<PoolSize>800</PoolSize>
</TransportThreadPool>
<IdleWorkers>
<WorkerCount>16</WorkerCount>
<CheckFrequency>50</CheckFrequency>
<MinimumWaitTime>5</MinimumWaitTime>
</IdleWorkers>
<NetConnections>
<ProcessorCount>16</ProcessorCount>
<IdleFrequency>250</IdleFrequency>
<SocketConfiguration>
<ReuseAddress>false</ReuseAddress>
<ReceiveBufferSize>131072</ReceiveBufferSize>
<ReadBufferSize>131072</ReadBufferSize>
<SendBufferSize>131072</SendBufferSize>
<KeepAlive>true</KeepAlive>
<AcceptorBackLog>100</AcceptorBackLog>
</SocketConfiguration>
</NetConnections>
<HTTPTunnel>
<KeepAliveTimeout>2000</KeepAliveTimeout>
</HTTPTunnel>
<Client>
<ClientTimeout>9000</ClientTimeout>
<IdleFrequency>250</IdleFrequency>
</Client>
<RTP>
<IdleFrequency>75</IdleFrequency>
<DatagramConfiguration>
<Incoming>
<ReuseAddress>true</ReuseAddress>
<ReceiveBufferSize>131072</ReceiveBufferSize>
<SendBufferSize>131072</SendBufferSize>
<MulticastTimeout>50</MulticastTimeout>
<DatagramMaximumPacketSize>4096</DatagramMaximumPacketSize>
</Incoming>
<Outgoing>
<ReuseAddress>false</ReuseAddress>
<ReceiveBufferSize>131072</ReceiveBufferSize>
<SendBufferSize>131072</SendBufferSize>
<MulticastTimeout>50</MulticastTimeout>
<DatagramMaximumPacketSize>4096</DatagramMaximumPacketSize>
</Outgoing>
</DatagramConfiguration>
<UnicastIncoming>
<ProcessorCount>8</ProcessorCount>
</UnicastIncoming>
<UnicastOutgoing>
<ProcessorCount>16</ProcessorCount>
</UnicastOutgoing>
<MulticastIncoming>
<ProcessorCount>8</ProcessorCount>
</MulticastIncoming>
<MulticastOutgoing>
<ProcessorCount>8</ProcessorCount>
</MulticastOutgoing>
</RTP>
<Application>
<ApplicationTimeout>60000</ApplicationTimeout>
<PingTimeout>12000</PingTimeout>
<ValidationFrequency>20000</ValidationFrequency>
<MaximumPendingWriteBytes>0</MaximumPendingWriteBytes>
<MaximumSetBufferTime>60000</MaximumSetBufferTime>
</Application>
<StartStartupStreams>true</StartStartupStreams>
<Properties>
</Properties>
</VHost>
</Root>
Someone have some ideas ?