残酷的OOM

问题： 30个虚拟机一下子全死光光了，而且/var/log/message 中没有任何遗言；莫不是被攻击了？

不要瞎猜，查一下吧

1. 因为rsyslogd死的比较早，所以 /var/log/message 没有任何有用的信息
2. 虽然mysqld被杀了很多次，但是依然存在，是因为mysqld_safe 没有被杀，当mysqld死掉时，mysqld_safe 就会再起一个mysqld。什么？mysqld_safe 也在死亡名单中出现了？确实，后来发现了，手动重启了，且看mysqld_safe 之后的多次mysqld被杀就行了
3. 终于VBoxSVC 被杀,所有虚拟机就全部死光光了
4. 如果把sshd杀死了，岂不是很悲催？ 参看openssh-server 的源码： https://github.com/openssh/openssh-portable/blob/8408218c1ca88cb17d15278174a24a94a6f65fe1/openbsd-compat/port-linux.c ，其中有如下逻辑：（就是说，这家伙有自我保护功能，通过调整/proc/self/oom_score_adj 或 /proc/self/oom_adj实现，其中的设置保证该进程不会被oom掉，其他进程也可以这么效仿）
   
   #ifdef LINUX_OOM_ADJUST /* * The magic "don't kill me" values, old and new, as documented in eg: * http://lxr.linux.no/#linux+v2.6.32/Documentation/filesystems/proc.txt * http://lxr.linux.no/#linux+v2.6.36/Documentation/filesystems/proc.txt */ static int oom_adj_save = INT_MIN; static char *oom_adj_path = NULL; struct { char *path; int value; } oom_adjust[] = { {"/proc/self/oom_score_adj", -1000}, /* kernels >= 2.6.36 */ {"/proc/self/oom_adj", -17}, /* kernels <= 2.6.35 */ {NULL, 0}, }; /* * Tell the kernel's out-of-memory killer to avoid sshd. * Returns the previous oom_adj value or zero. */ void oom_adjust_setup(void) { int i, value; FILE *fp; debug3("%s", __func__); for (i = 0; oom_adjust[i].path != NULL; i++) { oom_adj_path = oom_adjust[i].path; value = oom_adjust[i].value; if ((fp = fopen(oom_adj_path, "r+")) != NULL) { if (fscanf(fp, "%d", &oom_adj_save) != 1) verbose("error reading %s: %s", oom_adj_path, strerror(errno)); else { rewind(fp); if (fprintf(fp, "%d\n", value) <= 0) verbose("error writing %s: %s", oom_adj_path, strerror(errno)); else debug("Set %s from %d to %d", oom_adj_path, oom_adj_save, value); } fclose(fp); return; } } oom_adj_path = NULL; } /* Restore the saved OOM adjustment */ void oom_adjust_restore(void) { FILE *fp; debug3("%s", __func__); if (oom_adj_save == INT_MIN || oom_adj_path == NULL || (fp = fopen(oom_adj_path, "w")) == NULL) return; if (fprintf(fp, "%d\n", oom_adj_save) <= 0) verbose("error writing %s: %s", oom_adj_path, strerror(errno)); else debug("Set %s to %d", oom_adj_path, oom_adj_save); fclose(fp); return; } #endif /* LINUX_OOM_ADJUST */

   1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72

   #ifdef LINUX_OOM_ADJUST /* * The magic "don't kill me" values, old and new, as documented in eg: * http://lxr.linux.no/#linux+v2.6.32/Documentation/filesystems/proc.txt * http://lxr.linux.no/#linux+v2.6.36/Documentation/filesystems/proc.txt */   static int oom_adj_save = INT_MIN; static char *oom_adj_path = NULL; struct { char *path; int value; } oom_adjust[] = { {"/proc/self/oom_score_adj", -1000}, /* kernels >= 2.6.36 */ {"/proc/self/oom_adj", -17}, /* kernels <= 2.6.35 */ {NULL, 0}, };   /* * Tell the kernel's out-of-memory killer to avoid sshd. * Returns the previous oom_adj value or zero. */ void oom_adjust_setup(void) { int i, value; FILE *fp;   debug3("%s", __func__); for (i = 0; oom_adjust[i].path != NULL; i++) { oom_adj_path = oom_adjust[i].path; value = oom_adjust[i].value; if ((fp = fopen(oom_adj_path, "r+")) != NULL) { if (fscanf(fp, "%d", &oom_adj_save) != 1) verbose("error reading %s: %s", oom_adj_path,     strerror(errno)); else { rewind(fp); if (fprintf(fp, "%d\n", value) <= 0) verbose("error writing %s: %s",    oom_adj_path, strerror(errno)); else debug("Set %s from %d to %d",    oom_adj_path, oom_adj_save, value); } fclose(fp); return; } } oom_adj_path = NULL; }   /* Restore the saved OOM adjustment */ void oom_adjust_restore(void) { FILE *fp;   debug3("%s", __func__); if (oom_adj_save == INT_MIN || oom_adj_path == NULL ||     (fp = fopen(oom_adj_path, "w")) == NULL) return;   if (fprintf(fp, "%d\n", oom_adj_save) <= 0) verbose("error writing %s: %s", oom_adj_path, strerror(errno)); else debug("Set %s to %d", oom_adj_path, oom_adj_save);   fclose(fp); return; } #endif /* LINUX_OOM_ADJUST */

   
    

参考资料： https://www.kernel.org/doc/Documentation/filesystems/proc.txt