Re: [vserver] File System Full

From: Christian Thaeter <ct_at_pipapo.org>
Date: Fri 10 Feb 2012 - 11:13:49 GMT
Message-ID: <20120210121349.66a336ec@pluto.pipapo.org>

Am Fri, 10 Feb 2012 10:45:00 +0000 (GMT)
schrieb Phil Daws <uxbod@splatnix.net>:

> Hello all,
>
>
> we are seeing a strange issue where one of our vserver guests is
> reporting that its file system is full yet there is plenty of space:

I had that problem too (with some very old vserver kernel), upgrading to
a new kernel fixed it, at least it never happened to me again.

Symptoms where like you described, FS is reported full, space and
inodes where not fully used, restarting the vserver in question fixed it
for a while but it will reappear sooner or later, eventually this also
happened on the root server.

        Christian

>
>
>
> usera:/var/run# df -h
> Filesystem Size Used Avail Use% Mounted on
> /dev/hdv1 1008G 187G 812G 19% /
> none 128M 0 128M 0% /tmp
> usera:/var/run# touch testfile
> touch: cannot touch `testfile': No space left on device
>
>
> Below is a strace -f when running the touch testfile command:
>
>
>
> usera:/var/run# strace -f touch testfile
> execve("/usr/bin/touch", ["touch", "testfile"], [/* 28 vars */]) = 0
> brk(0) = 0x60d000
> access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or
> directory) mmap(NULL, 8192, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3429ad3000
> access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or
> directory) open("/etc/ld.so.cache", O_RDONLY) = 3 fstat(3,
> {st_mode=S_IFREG|0644, st_size=153901, ...}) = 0 mmap(NULL, 153901,
> PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f3429aad000 close(3) = 0
> access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or
> directory) open("/lib/librt.so.1", O_RDONLY) = 3
> read(3,
> "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220!\0\0\0\0\0\0@"...,
> 832) = 832 fstat(3, {st_mode=S_IFREG|0644, st_size=31744, ...}) = 0
> mmap(NULL, 2128848, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE,
> 3, 0) = 0x7f34296b0000 mprotect(0x7f34296b7000, 2093056, PROT_NONE) =
> 0 mmap(0x7f34298b6000, 8192, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7f34298b6000
> close(3) = 0 access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such
> file or directory) open("/lib/libc.so.6", O_RDONLY) = 3 read(3,
> "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`\355\1\0\0\0\0\0@"...,
> 832) = 832 fstat(3, {st_mode=S_IFREG|0755, st_size=1432968, ...}) = 0
> mmap(NULL, 3541032, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE,
> 3, 0) = 0x7f342934f000 mprotect(0x7f34294a7000, 2093056, PROT_NONE) =
> 0 mmap(0x7f34296a6000, 20480, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x157000) = 0x7f34296a6000
> mmap(0x7f34296ab000, 18472, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f34296ab000 close(3)
> = 0 access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or
> directory) open("/lib/libpthread.so.0", O_RDONLY) = 3 read(3,
> "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340Y\0\0\0\0\0\0@"...,
> 832) = 832 fstat(3, {st_mode=S_IFREG|0755, st_size=131260, ...}) = 0
> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
> 0) = 0x7f3429aac000 mmap(NULL, 2208640, PROT_READ|PROT_EXEC,
> MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f3429133000
> mprotect(0x7f342914a000, 2093056, PROT_NONE) = 0 mmap(0x7f3429349000,
> 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3,
> 0x16000) = 0x7f3429349000 mmap(0x7f342934b000, 13184,
> PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) =
> 0x7f342934b000 close(3) = 0 mmap(NULL, 4096, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3429aab000 mmap(NULL, 4096,
> PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
> 0x7f3429aaa000 arch_prctl(ARCH_SET_FS, 0x7f3429aab700) = 0
> open("/dev/urandom", O_RDONLY) = 3 read(3, "\270_f\37!\232;"..., 7) =
> 7 close(3) = 0 mprotect(0x7f3429349000, 4096, PROT_READ) = 0
> mprotect(0x7f34296a6000, 16384, PROT_READ) = 0
> mprotect(0x7f34298b6000, 4096, PROT_READ) = 0
> mprotect(0x7f3429ad5000, 4096, PROT_READ) = 0 munmap(0x7f3429aad000,
> 153901) = 0 set_tid_address(0x7f3429aab9d0) = 13524
> set_robust_list(0x7f3429aab9e0, 0x18) = 0 futex(0x7fff31ad739c,
> FUTEX_WAKE_PRIVATE, 1) = 0 futex(0x7fff31ad739c, 0x189 /* FUTEX_???
> */, 1, NULL, 7f3429aab700) = -1 ENOSYS (Function not implemented)
> rt_sigaction(SIGRTMIN, {0x7f3429138860, [], SA_RESTORER|SA_SIGINFO,
> 0x7f3429141f60}, NULL, 8) = 0 rt_sigaction(SIGRT_1, {0x7f34291388f0,
> [], SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x7f3429141f60}, NULL, 8) = 0
> rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
> getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY})
> = 0 brk(0) = 0x60d000 brk(0x62e000) = 0x62e000
> open("/usr/lib/locale/locale-archive", O_RDONLY) = 3 fstat(3,
> {st_mode=S_IFREG|0644, st_size=85880592, ...}) = 0 mmap(NULL,
> 85880592, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f3423f4c000 close(3) = 0
> open("testfile", O_WRONLY|O_CREAT|O_NOCTTY|O_NONBLOCK, 0666) = -1
> ENOSPC (No space left on device) futimesat(AT_FDCWD, "testfile",
> NULL) = -1 ENOENT (No such file or directory)
> open("/usr/share/locale/locale.alias", O_RDONLY) = 3 fstat(3,
> {st_mode=S_IFREG|0644, st_size=2570, ...}) = 0 mmap(NULL, 4096,
> PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
> 0x7f3429ad2000 read(3, "# Locale name alias data base.\n# "..., 4096)
> = 2570 read(3, ""..., 4096) = 0 close(3) = 0
> munmap(0x7f3429ad2000, 4096) = 0
> open("/usr/share/locale/en_US.UTF-8/LC_MESSAGES/coreutils.mo",
> O_RDONLY) = -1 ENOENT (No such file or directory)
> open("/usr/share/locale/en_US.utf8/LC_MESSAGES/coreutils.mo",
> O_RDONLY) = -1 ENOENT (No such file or directory)
> open("/usr/share/locale/en_US/LC_MESSAGES/coreutils.mo", O_RDONLY) =
> -1 ENOENT (No such file or directory)
> open("/usr/share/locale/en.UTF-8/LC_MESSAGES/coreutils.mo", O_RDONLY)
> = -1 ENOENT (No such file or directory)
> open("/usr/share/locale/en.utf8/LC_MESSAGES/coreutils.mo", O_RDONLY)
> = -1 ENOENT (No such file or directory)
> open("/usr/share/locale/en/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1
> ENOENT (No such file or directory) write(2, "touch: "..., 7touch: ) =
> 7 write(2, "cannot touch `testfile'"..., 23cannot touch `testfile') =
> 23 open("/usr/share/locale/en_US.UTF-8/LC_MESSAGES/libc.mo",
> O_RDONLY) = -1 ENOENT (No such file or directory)
> open("/usr/share/locale/en_US.utf8/LC_MESSAGES/libc.mo", O_RDONLY) =
> -1 ENOENT (No such file or directory)
> open("/usr/share/locale/en_US/LC_MESSAGES/libc.mo", O_RDONLY) = -1
> ENOENT (No such file or directory)
> open("/usr/share/locale/en.UTF-8/LC_MESSAGES/libc.mo", O_RDONLY) = -1
> ENOENT (No such file or directory)
> open("/usr/share/locale/en.utf8/LC_MESSAGES/libc.mo", O_RDONLY) = -1
> ENOENT (No such file or directory)
> open("/usr/share/locale/en/LC_MESSAGES/libc.mo", O_RDONLY) = -1
> ENOENT (No such file or directory) write(2, ": No space left on
> device"..., 25: No space left on device) = 25 write(2, "\n"..., 1 ) =
> 1 close(1) = 0 close(2) = 0 exit_group(1) = ?
>
>
> Any thoughts as I have checked the host system and that looks
> absolutely fine; and so are the other vservers. Weird! :)
Received on Fri Feb 10 11:14:00 2012

[Next/Previous Months] [Main vserver Project Homepage] [Howto Subscribe/Unsubscribe] [Paul Sladen's vserver stuff]
Generated on Fri 10 Feb 2012 - 11:14:00 GMT by hypermail 2.1.8