Agent won't start.

[Wido]

Hi,

I have made my own Zabbix Agent packages voor Debian and Ubuntu. They work great, but on one of my servers, the Zabbix agent won't start.

When i do a strace, i get:

execve("/usr/local/bin/zabbix_agentd", ["/usr/local/bin/zabbix_agentd"], [/* 15 vars */]) = 0
uname({sys="Linux", node="reseller08.pcextreme.nl", ...}) = 0
brk(0)                                  = 0x80b2000
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x5556c000
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/etc/ld.so.preload", O_RDONLY)    = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY)      = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=16458, ...}) = 0
old_mmap(NULL, 16458, PROT_READ, MAP_PRIVATE, 3, 0) = 0x5556d000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/tls/libresolv.so.2", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220)\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0644, st_size=64924, ...}) = 0
old_mmap(NULL, 73640, PROT_READ|PROT_EXEC, MAP_PRIVATE, 3, 0) = 0x55572000
old_mmap(0x55581000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3, 0xf000) = 0x55581000
old_mmap(0x55582000, 8104, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x55582000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/tls/libnsl.so.1", O_RDONLY)  = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0 <\0\000"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0644, st_size=73304, ...}) = 0
old_mmap(NULL, 80544, PROT_READ|PROT_EXEC, MAP_PRIVATE, 3, 0) = 0x55584000
old_mmap(0x55595000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3, 0x11000) = 0x55595000
old_mmap(0x55596000, 6816, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x55596000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/tls/libc.so.6", O_RDONLY)    = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`Z\1\000"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=1254660, ...}) = 0
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x55598000
old_mmap(NULL, 1264972, PROT_READ|PROT_EXEC, MAP_PRIVATE, 3, 0) = 0x55599000
old_mmap(0x556c3000, 36864, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3, 0x129000) = 0x556c3000
old_mmap(0x556cc000, 7500, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x556cc000
close(3)                                = 0
set_thread_area({entry_number:-1 -> 12, base_addr:0x55598940, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}) = 0
munmap(0x5556d000, 16458)               = 0
brk(0)                                  = 0x80b2000
brk(0x80d3000)                          = 0x80d3000
brk(0)                                  = 0x80d3000
open("/etc/zabbix/zabbix_agentd.conf", O_RDONLY) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=121, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x5556d000
read(3, "Server=217.115.199.215\nHostname="..., 4096) = 121
read(3, "", 4096)                       = 0
open("/etc/zabbix/zabbix_agentd.conf", O_RDONLY) = 4
fstat64(4, {st_mode=S_IFREG|0644, st_size=121, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x5556e000
read(4, "Server=217.115.199.215\nHostname="..., 4096) = 121
read(4, "", 4096)                       = 0
getuid32()                              = 0
socket(PF_FILE, SOCK_STREAM, 0)         = 5
connect(5, {sa_family=AF_FILE, path="/var/run/.nscd_socket"}, 110) = -1 ENOENT (No such file or directory)
close(5)                                = 0
open("/etc/nsswitch.conf", O_RDONLY)    = 5
fstat64(5, {st_mode=S_IFREG|0644, st_size=465, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x5556f000
read(5, "# /etc/nsswitch.conf\n#\n# Example"..., 4096) = 465
read(5, "", 4096)                       = 0
close(5)                                = 0
munmap(0x5556f000, 4096)                = 0
open("/etc/ld.so.cache", O_RDONLY)      = 5
fstat64(5, {st_mode=S_IFREG|0644, st_size=16458, ...}) = 0
old_mmap(NULL, 16458, PROT_READ, MAP_PRIVATE, 5, 0) = 0x556ce000
close(5)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/tls/libnss_compat.so.2", O_RDONLY) = 5
read(5, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0000\22\0"..., 512) = 512
fstat64(5, {st_mode=S_IFREG|0644, st_size=28616, ...}) = 0
old_mmap(NULL, 31628, PROT_READ|PROT_EXEC, MAP_PRIVATE, 5, 0) = 0x556d3000
old_mmap(0x556da000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 5, 0x6000) = 0x556da000
close(5)                                = 0
munmap(0x556ce000, 16458)               = 0
open("/etc/ld.so.cache", O_RDONLY)      = 5
fstat64(5, {st_mode=S_IFREG|0644, st_size=16458, ...}) = 0
old_mmap(NULL, 16458, PROT_READ, MAP_PRIVATE, 5, 0) = 0x556ce000
close(5)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/tls/libnss_nis.so.2", O_RDONLY) = 5
read(5, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\20\34\0"..., 512) = 512
fstat64(5, {st_mode=S_IFREG|0644, st_size=33440, ...}) = 0
old_mmap(NULL, 36620, PROT_READ|PROT_EXEC, MAP_PRIVATE, 5, 0) = 0x556db000
old_mmap(0x556e3000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 5, 0x7000) = 0x556e3000
close(5)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/tls/libnss_files.so.2", O_RDONLY) = 5
read(5, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\200\35"..., 512) = 512
fstat64(5, {st_mode=S_IFREG|0644, st_size=34748, ...}) = 0
old_mmap(NULL, 38044, PROT_READ|PROT_EXEC, MAP_PRIVATE, 5, 0) = 0x556e4000
old_mmap(0x556ed000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 5, 0x8000) = 0x556ed000
close(5)                                = 0
munmap(0x556ce000, 16458)               = 0
open("/etc/passwd", O_RDONLY)           = 5
fcntl64(5, F_GETFD)                     = 0
fcntl64(5, F_SETFD, FD_CLOEXEC)         = 0
_llseek(5, 0, [0], SEEK_CUR)            = 0
fstat64(5, {st_mode=S_IFREG|0644, st_size=8070, ...}) = 0
mmap2(NULL, 8070, PROT_READ, MAP_SHARED, 5, 0) = 0x5556f000
_llseek(5, 8070, [8070], SEEK_SET)      = 0
munmap(0x5556f000, 8070)                = 0
close(5)                                = 0
setgid32(32000)                         = 0
setuid32(32000)                         = 0
setresgid32(-1, 32000, -1)              = 0
setresuid32(-1, 32000, -1)              = 0
clone(child_stack=0, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0x55598988) = 31038
exit_group(0)

reseller08:~# /usr/local/bin/zabbix_agentd -v
ZABBIX Agent (daemon) v1.1.2 (19 July 2006)
Compilation time:  Sep 18 2006 18:05:58
reseller08:~#

The strace looks fine to me, but still, the Zabbix agent won't start.

My zabbix_agentd.conf looks like:

Server=217.115.199.215
Hostname=localhost
StartAgents=5
DebugLevel=3
PidFile=/var/run/zabbix/zabbix_agentd.pid
Timeout=5

Any ideas?

[Wido]

Little kick?

[den_crane]

...but on one of my servers, the Zabbix agent won't start.
...

uname -a
may be server has different platform (for example 64bit)

[Wido]

uname -a
may be server has different platform (for example 64bit)

Linux reseller08.pcextreme.nl 2.6.15-27-amd64-k8 #1 SMP PREEMPT Thu Jul 20 15:
:29 CEST 2006 x86_64 GNU/Linux
reseller08:~#

A little side note, it worked before on this server.

This is a Linux VServer guest (Debian Sarge 32Bits) running on a Ubuntu Dapper (AMD64) system.

The agent worked, but it justed stopped working after a few weeks.

The rest of the services runs fine, it is only Zabbix wich is giving me troubles.

[Wido]

Well, i still got this problem.

Currently i am running 1.1.4 and i have this problem on 6 Debian Sarge servers.

apt-get update && apt-get upgrade has been done, all the libraries are up to date.

Some debug information:

reseller01:~# file /usr/local/bin/zabbix_agentd
/usr/local/bin/zabbix_agentd: ELF 32-bit LSB executable, Intel 80386, version 1 (SYSV), for GNU/Linux 2.2.0, dynamically linked (uses shared libs), not stripped
reseller01:~# uname -a
Linux reseller01.pcextreme.nl 2.6.8-3-686-smp #1 SMP Thu Sep 7 04:39:15 UTC 2006 i686 GNU/Linux
reseller01:~#

reseller08:~# file /usr/local/bin/zabbix_agentd
/usr/local/bin/zabbix_agentd: ELF 32-bit LSB executable, Intel 80386, version 1 (SYSV), for GNU/Linux 2.2.0, dynamically linked (uses shared libs), not stripped
reseller08:~# uname -a
Linux reseller08.pcextreme.nl 2.6.16.29-xen #3 SMP Sun Oct 15 13:15:34 BST 2006 x86_64 GNU/Linux
reseller08:~#

On both servers the Zabbix Agent will not start.

ZABBIX Agent (daemon) v1.1.4 (14 November 2006)

And again, the output of strace

reseller01:~# strace zabbix_agentd
execve("/usr/local/bin/zabbix_agentd", ["zabbix_agentd"], [/* 18 vars */]) = 0
uname({sys="Linux", node="reseller01.pcextreme.nl", ...}) = 0
brk(0)                                  = 0x80b2000
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x40017000
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/etc/ld.so.preload", O_RDONLY)    = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=16, ...}) = 0
old_mmap(NULL, 16, PROT_READ|PROT_WRITE, MAP_PRIVATE, 3, 0) = 0x40018000
close(3)                                = 0
open("/lib/snoopy.so", O_RDONLY)        = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\0\10\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0644, st_size=4616, ...}) = 0
old_mmap(NULL, 7696, PROT_READ|PROT_EXEC, MAP_PRIVATE, 3, 0) = 0x40019000
old_mmap(0x4001a000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3, 0) = 0x4001a000
close(3)                                = 0
munmap(0x40018000, 16)                  = 0
open("/etc/ld.so.cache", O_RDONLY)      = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=17888, ...}) = 0
old_mmap(NULL, 17888, PROT_READ, MAP_PRIVATE, 3, 0) = 0x4001b000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/tls/libresolv.so.2", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220)\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0644, st_size=64924, ...}) = 0
old_mmap(NULL, 73640, PROT_READ|PROT_EXEC, MAP_PRIVATE, 3, 0) = 0x40020000
old_mmap(0x4002f000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3, 0xf000) = 0x4002f000
old_mmap(0x40030000, 8104, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x40030000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/tls/libnsl.so.1", O_RDONLY)  = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0 <\0\000"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0644, st_size=73304, ...}) = 0
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x40032000
old_mmap(NULL, 80544, PROT_READ|PROT_EXEC, MAP_PRIVATE, 3, 0) = 0x40033000
old_mmap(0x40044000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3, 0x11000) = 0x40044000
old_mmap(0x40045000, 6816, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x40045000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/tls/libc.so.6", O_RDONLY)    = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`Z\1\000"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=1254660, ...}) = 0
old_mmap(NULL, 1264972, PROT_READ|PROT_EXEC, MAP_PRIVATE, 3, 0) = 0x40047000
old_mmap(0x40171000, 36864, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3, 0x129000) = 0x40171000
old_mmap(0x4017a000, 7500, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x4017a000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/tls/libdl.so.2", O_RDONLY)   = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\320\32"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0644, st_size=9872, ...}) = 0
old_mmap(NULL, 8632, PROT_READ|PROT_EXEC, MAP_PRIVATE, 3, 0) = 0x4017c000
old_mmap(0x4017e000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3, 0x2000) = 0x4017e000
close(3)                                = 0
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x4017f000
set_thread_area({entry_number:-1 -> 6, base_addr:0x4017f080, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}) = 0
munmap(0x4001b000, 17888)               = 0
brk(0)                                  = 0x80b2000
brk(0x80d3000)                          = 0x80d3000
brk(0)                                  = 0x80d3000
open("/etc/zabbix/zabbix_agentd.conf", O_RDONLY) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=121, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x4001b000
read(3, "Server=217.115.199.215\nHostname="..., 4096) = 121
read(3, "", 4096)                       = 0
open("/etc/zabbix/zabbix_agentd.conf", O_RDONLY) = 4
fstat64(4, {st_mode=S_IFREG|0644, st_size=121, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x4001c000
read(4, "Server=217.115.199.215\nHostname="..., 4096) = 121
read(4, "", 4096)                       = 0
getuid32()                              = 0
socket(PF_FILE, SOCK_STREAM, 0)         = 5
connect(5, {sa_family=AF_FILE, path="/var/run/.nscd_socket"}, 110) = -1 ENOENT (No such file or directory)
close(5)                                = 0
open("/etc/nsswitch.conf", O_RDONLY)    = 5
fstat64(5, {st_mode=S_IFREG|0644, st_size=465, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x4001d000
read(5, "# /etc/nsswitch.conf\n#\n# Example"..., 4096) = 465
read(5, "", 4096)                       = 0
close(5)                                = 0
munmap(0x4001d000, 4096)                = 0
open("/etc/ld.so.cache", O_RDONLY)      = 5
fstat64(5, {st_mode=S_IFREG|0644, st_size=17888, ...}) = 0
old_mmap(NULL, 17888, PROT_READ, MAP_PRIVATE, 5, 0) = 0x40180000
close(5)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/tls/libnss_compat.so.2", O_RDONLY) = 5
read(5, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0000\22\0"..., 512) = 512
fstat64(5, {st_mode=S_IFREG|0644, st_size=28616, ...}) = 0
old_mmap(NULL, 31628, PROT_READ|PROT_EXEC, MAP_PRIVATE, 5, 0) = 0x40185000
old_mmap(0x4018c000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 5, 0x6000) = 0x4018c000
close(5)                                = 0
munmap(0x40180000, 17888)               = 0
open("/etc/ld.so.cache", O_RDONLY)      = 5
fstat64(5, {st_mode=S_IFREG|0644, st_size=17888, ...}) = 0
old_mmap(NULL, 17888, PROT_READ, MAP_PRIVATE, 5, 0) = 0x40180000
close(5)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/tls/libnss_nis.so.2", O_RDONLY) = 5
read(5, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\20\34\0"..., 512) = 512
fstat64(5, {st_mode=S_IFREG|0644, st_size=33440, ...}) = 0
old_mmap(NULL, 36620, PROT_READ|PROT_EXEC, MAP_PRIVATE, 5, 0) = 0x4018d000
old_mmap(0x40195000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 5, 0x7000) = 0x40195000
close(5)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/tls/libnss_files.so.2", O_RDONLY) = 5
read(5, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\200\35"..., 512) = 512
fstat64(5, {st_mode=S_IFREG|0644, st_size=34748, ...}) = 0
old_mmap(NULL, 38044, PROT_READ|PROT_EXEC, MAP_PRIVATE, 5, 0) = 0x40196000
old_mmap(0x4019f000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 5, 0x8000) = 0x4019f000
close(5)                                = 0
munmap(0x40180000, 17888)               = 0
open("/etc/passwd", O_RDONLY)           = 5
fcntl64(5, F_GETFD)                     = 0
fcntl64(5, F_SETFD, FD_CLOEXEC)         = 0
_llseek(5, 0, [0], SEEK_CUR)            = 0
fstat64(5, {st_mode=S_IFREG|0644, st_size=19350, ...}) = 0
mmap2(NULL, 19350, PROT_READ, MAP_SHARED, 5, 0) = 0x40180000
_llseek(5, 19350, [19350], SEEK_SET)    = 0
munmap(0x40180000, 19350)               = 0
close(5)                                = 0
setgid32(32000)                         = 0
setuid32(32000)                         = 0
setresgid32(-1, 32000, -1)              = 0
setresuid32(-1, 32000, -1)              = 0
clone(child_stack=0, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0x4017f0c8) = 17045
--- SIGCHLD (Child exited) @ 0 (0) ---
exit_group(0)                           = ?
reseller01:~#

It really starts to annoy me, what is going wrong here?

[jeff.showers]

Can you change your DebugLevel to 4 and see if that gives any more meaningful error data to work with?

[Wido]

Yes, it resolved it.

I had to remove all the zabbix* files in /tmp, after removing those it worked again.

Really, why didn't i figure this out earlier?

[jeff.showers]

No worries. Lord knows we all get caught up in a problem and start to overlook some things at some point. =)