Terapix (archive) - Intranet

formating RAID5 /dev/sdb on mix2

# mkfs -V -t xfs -f -d su=64k,sw=8 -L "raid1" /dev/sdb

On mix3 (12 disks):

# mkfs -V -t xfs -f -d su=64k,sw=12 -L "raid1" /dev/sdb

optimization xfs disks (big readahead for large files storage)

# vi /etc/conf.d/local.start /sbin/blockdev --setra 2048 /dev/sdb /sbin/blockdev --setra 2048 /dev/sdc

tcp transfer optomization Cf. this page or this one

# vi /etc/conf.d/local.start echo 4096 87380 16777216 >> /proc/sys/net/ipv4/tcp_rmem echo 4096 87380 16777216 >> /proc/sys/net/ipv4/tcp_wmem echo 16777216 >> /proc/sys/net/core/wmem_max echo 16777216 >> /proc/sys/net/core/rmem_max

or better: # sysctl -a | egrep -e 'tcp_rmem|tcp_wmem|wmem_max|rmem_max' # vi /etc/sysctl.conf # TCP tuning cf. http://www-didc.lbl.gov/TCP-tuning/linux.html # increase TCP max buffer size net.core.rmem_max = 16777216 net.core.wmem_max = 16777216 # increase Linux autotuning TCP buffer limits # min, default, and max number of bytes to use net.ipv4.tcp_rmem = 4096 87380 16777216 net.ipv4.tcp_wmem = 4096 65536 16777216 # don't cache ssthresh from previous connection (default: 0) net.ipv4.tcp_no_metrics_save = 1 # recommended to increase this for 1000 BT or higher (default: 300) net.core.netdev_max_backlog = 2500

32bits emulation (obsolete)

# emerge app-emulation/emul-linux-x86-baselibs # emerge app-emulation/emul-linux-x86-xlibs # emerge app-emulation/emul-linux-x86-gtklibs

add missing 32bits libraries from manix:

# cd /lib # rsync -av libtermcap.so* root@efigix:/lib32/ # scp /usr/local/lib/libcfitsio.so root@efigix:/lib32/

/etc/profile

# vi /etc/profile [ -f /etc/profile.d/bash-completion ] && source /etc/profile.d/bash-completion # *** FM *** eval `dircolors -b /etc/DIR_COLORS` opt='--color=auto' alias ls="/bin/ls $opt" alias l="/bin/ls $opt -FaC" alias ll="/bin/ls $opt -Fals " alias j='jobs' alias h='history' alias mv='mv -vi' alias grep='grep --color=auto' alias mountraids='ls -l /data/clix/fc*/{BROWNIE,STOREX}* /data/f*ix*/raid{,1,2}/this_is*' export TIMEFORMAT='%U user %S system %R elapsed %P%% CPU' export TIME='%U user %S system %e elapsed %P CPU' export HISTTIMEFORMAT="%F %T # " alias vi='vim -X' alias view='view -X' locebuild () { locate $1 | grep ebuild } machines_opterons=(mix{1,2,3,4,5,6,7} clix dbterapix ftpix fcix cosmix{2,3,4}) machines_opterons_bipro=(mix{1,2} clix dbterapix ftpix fcix cosmix{2,4}) machines_opterons_quadripro=(mix{3,4,5,6,7} cosmix3) machines_opterons_clix=(mix{1,2,3,4,5,6,7} clix dbterapix ftpix fcix) machines_opterons_cosmix=(cosmix{2,3,4}) machines_cluster_clic=(clix pix{1,2,3,4,5,6,7,8,9} mix{1,2,3,4,5,6,7} dbterapix ftpix fcix) machines_x86_clic=(pix{1,2,3,4,5,6,7,8,9}) compare () { for machine in ${machines_cluster_clic[*]}; do echo -e "\n$machine" ssh $machine diff $1 $2 done } ulimit -n 1048576

/etc/csh.cshrc: is now OK ! make sure default path are prepended, and not appended

if ($LOGNAME == "root") then set -f path = ( /sbin $path ) set -f path = ( /usr/sbin $path ) set -f path = ( /usr/local/sbin $path ) endif set -f path = ( /bin $path ) set -f path = ( /usr/bin $path ) set -f path = ( /usr/local/bin $path ) set -f path = ( /opt/bin $path )

dispatch-conf

# vi /etc/dispatch-conf.conf archive-dir=/etc/config-archive log-file=/var/log/dispatch-conf.log # mkdir /etc/config-archive

/etc/sysctl.conf

# vi /etc/sysctl.conf kernel.panic = 3

kernel recompilation:

recompile with the following changes to files /usr/src/linux/include/linux/fs.h and /usr/include/linux/fs.h:

#define INR_OPEN (1024*1024) #define NR_FILE 32768

/.vimrc

simple:

# scp mix1:~/.vimrc ~/; scp mix1:~/.gvimrc ~/

less simple:

# cat >> ~/.vimrc syntax on "Default backspace like normal set bs=2 "Show matching parenthese. set showmatch set showcmd set showmode set laststatus=2 set ruler " do not force to write when change buffer ! set hidden set formatoptions=cqr2 set hlsearch set incsearch map u :set hls! " FM : <C-O> start a block imap <C-O> {<CR>}<Esc>O nmap Q gq vmap Q gq noremap <C-G> 2<C-G> # cat >> ~/.vimrc set guioptions=gimrtT set mouse=v set ch=2 map <S-Insert> <MiddleMouse> map! <S-Insert> <MiddleMouse> set clipboard^=autoselectml guioptions+=A

Bug in imagemagick configuration (/usr/lib/ImageMagick-6.1.0/config/type-ghostscript.mgk): fonts are not in the right directory. Let's make a symlink to make them visible (necessary for convert in QF)

cd /usr/share/ghostscript ln -s /usr/share/fonts/default/ghostscript fonts

switch 2004.3 -> 2005.0

Cf. this page and this one on the forums

vi /etc/make.conf remove multilib USE="multilib" FEATURES="-sandbox" emerge --oneshot -va gcc emerge -va --oneshot '>=sys-apps/portage-2.0.51-r9' emerge -va --oneshot '>=sys-devel/gcc-config-1.3.9' emerge -va --oneshot '>=sys-devel/distcc-2.18.3-r1' emerge unmerge -va linux26-headers emerge -va --oneshot '>=linux-headers-2.6.8.1-r4' emerge -va --oneshot '>=emul-linux-x86-glibc-2.3.4.20041102' rm /lib32 /usr/lib32 mkdir /lib32 /usr/lib32 cp /emul/linux/x86/lib32/libsandbox.so /usr/lib32 ou cp /home/nis/root.nis/fred/opterons.config/libsandbox.so /usr/lib32 cp /emul/linux/x86/usr/lib32/libc.so /usr/lib32 cp /emul/linux/x86/usr/lib32/libpthread.so /usr/lib32 cp /emul/linux/x86/usr/lib32/*crt*.o /usr/lib32 env-update emerge -va --oneshot baselayout rm /etc/make.profile ln -s /usr/portage/profiles/default-linux/amd64/2005.0 /etc/make.profile USE="-nptl" emerge -va --oneshot '>=sys-libs/glibc-2.3.4.20041102' ou, aprÃ¨s le debuggage: emerge -va --oneshot '>=sys-libs/glibc-2.3.4.20041102' emerge unmerge -va emul-linux-x86-glibc rm -v /emul/linux/x86/usr/lib/libsandbox.* emerge -uva system

switch mix2 <-> mix4

AprÃ¨s le transfert, changer le nom de la machine dans:
/etc/fstab
arborescence /data
/etc/exports
adresse MAC dans /etc/dhcpd.conf sur clix
/etc/conf.d/net : changer IP
/etc/hostname
/etc/hosts

man.config

Ne pas utiliser MANPATH, mais modifier /etc/man.config

RAID1 configuration for mysql on dbterapix

read speed on individual disks:

# hdparm -v -tT /dev/sdb Timing cached reads: 4200 MB in 2.00 seconds = 2100.32 MB/sec Timing buffered disk reads: 200 MB in 3.02 seconds = 66.30 MB/sec # hdparm -v -tT /dev/sdc Timing cached reads: 4200 MB in 2.00 seconds = 2099.27 MB/sec Timing buffered disk reads: 200 MB in 3.02 seconds = 66.24 MB/sec

format (with cfdisk): keep 4GB (4096 MB) at the begining of the disk for swap space

partition size disk device
4GB = 4096MB /dev/sdb1
remaining /dev/sdb2

4GB = 4096MB /dev/sdc1
remaining /dev/sdc2

# mkswap /dev/sdb1 # mkswap /dev/sdc1 # vi /etc/fstab /dev/sdb1 none swap sw 0 0 /dev/sdc1 none swap sw 0 0

prepare software raid

# modprobe raid1 # vi /etc/raidtab raiddev /dev/md0 raid-level 1 nr-raid-disks 2 nr-spare-disks 0 chunk-size 4 persistent-superblock 1 device /dev/sdb2 raid-disk 0 device /dev/sdc2 raid-disk 1 # mkraid --really-force /dev/md0 # cat /proc/mdstat

make it usable for mysql

# mkreiserfs /dev/md0 # hdparm -v -tT /dev/md0 (while RAID constructing) Timing cached reads: 3916 MB in 2.00 seconds = 1956.34 MB/sec Timing buffered disk reads: 196 MB in 3.03 seconds = 64.78 MB/sec # mkdir /dbdisk # chmod 777 /dbdisk # vi /etc/fstab /dev/md0 /dbdisk reiserfs noatime 0 2 # mount /dbdisk # mkdir /dbdisk/mysql # vi /etc/mysql/my.cnf datadir = /dbdisk/mysql ### FM - TERAPIX ### #bind-address = 127.0.0.1 # /usr/bin/mysql_install_db # /etc/init.d/mysql start

Notification (cf. http://gentoo-wiki.com/HOWTO_Gentoo_Install_on_Software_RAID)

emerge mdadm vi /etc/mdadm.conf DEVICE /dev/sdb /dev/sdc ARRAY /dev/md0 devices=/dev/sdb,/dev/sdc MAILADDR magnard@iap.fr rc-update add mdadm default /etc/init.d/mdadm start mdadm -D /dev/md0 mdadm /dev/md0 -Gb internal watch -n .1 cat /proc/mdstat

Then remake mysql and phpmyadmin configuration like in this article.

Clock sync

now done at install

# vi /etc/conf.d/clock CLOCK="local" CLOCK_SYSTOHC="yes"

pix1: added e100 to kernel (in place of e1000 in modules list) # mix5: added vm.min_free_kbytes : 20000 in sysctl

cronjobs

# DO NOT EDIT THIS FILE - edit the master and reinstall. # (/tmp/crontab.XXXXiz7RSF installed on Mon Oct 30 16:10:06 2006) # (Cron version V5.0 -- $Id: crontab.c,v 1.12 2004/01/23 18:56:42 vixie Exp $) # update 3dm summary page every 5 min #0-59/5 * * * * /usr/local/bin/build_resume_3dm.sh 0-59/30 * * * * /root/scripts/build_3wraids_page.bash # check dead notes every 5 min */5 * * * * /usr/local/bin/down_node_checker.pl # ### FM ### backup of web and config 0 4,16 * * * if grep -q fc2 /proc/mounts; then /bin/nice -n 19 /usr/bin/rsync -q -av /var/www /etc /chroot /data/clix/fc2/backups_clix_nodelete/; fi # ### FM ### Sauvegarde mirroir de tout le disque (sauf les rep dynamiques comme /proc ou de RAID comme /data ou /mnt) # ### FM ### 30-Dec-2004: removed /dev and /tmp, added nice and condition on mounted device 0 10,22 * * * if grep -q mirrordisk /proc/mounts; then /bin/nice -n 19 /usr/bin/rsync -q -av --delete /bin /boot /chroot /emul /etc /home /lib /lib32 /lib64 /opt /root /sbin /usr /var /mirrordisk/; fi # ### FM ### watch too many processes events ?? #* * * * * (/usr/bin/date; /bin/ps auxf) >> /root/ps * * * * * n=`ps aux | grep ypserv | wc -l`; if [ $n -gt 10 ]; then echo "WARNING: some $n ypserv processes are running on clix"; strace -p `pgrep -o ypserv` 2>&1 | head -n 30; fi

add a new user

clix ~ # useradd -d /home/nis/fu -m -s /bin/tcsh fu clix ~ # passwd fu clix ~ # cd /var/yp/ clix yp # make

add a new group

# groupadd cosmo # usermod -G cosmo mellier # usermod -g cosmo fu # vi /home/nis/fu/.tcshrc umask 0002 # vi /home/nis/mellier/.tcshrc umask 0002 # cd /home/nis # chgrp -R -hc cosmo fu # chmod -R g+w fu # cd /data/mix1/raid2 # chown -R mellier:users yannick # chmod -R g+w yannick # cd /data/mix1/raid1/yannick # chown -R fu:cosmo Liping # cd /data/mix2/raid1/yannick/T0002/T0002sh # chown -R fu:cosmo Liping

file descriptors limit

# vi /etc/security/limits.conf * hard nofile 1048576 # vi /etc/ssh/sshd_config UsePAM yes # vi /etc/profile ulimit -n 1048576 # vi /etc/profile.d/terapix.csh limit descriptors 1048576 # chmod 755 /etc/profile.d/terapix.csh

initrd

An initrd file is simply a gzipped ext2 filesystem with insmod, nash acting as a busybox, and the necessary modules to load before init, everything being ordered by the linuxrc file. The directory /initrd *must* pre-exist (or pivot_root returns the very explicit error code 2)!

# mkinitrd --preload=3w-xxxx /boot/initrd.img-2.6.13 2.6.13

If the initrd is too big, the kernel compile-time parameter CONFIG_BLK_DEV_RAM_SIZE might need to be extended ! (default : 4096)

use mix8 as a local portage rsync server (cf. gentoo);

On the server (mix8): mix8 ~ # vi /etc/rsyncd.conf pid file = /var/run/rsyncd.pid max connections = 40 use chroot = yes uid = nobody gid = nobody read only = yes # allow terapix ip and intracluster network hosts allow = localhost 194.57.221.0/26 10.0.1.0/24 hosts deny = * # Simple example for enabling your own local rsync server [gentoo-portage] path = /usr/portage comment = Gentoo Portage tree exclude = distfiles/ packages/ mix8 ~ # /etc/init.d/rsyncd start mix8 ~ # rc-update add rsyncd default

on the hosts (efigix, mix9, fcix2, ...): efigix ~ # vi /etc/make.conf SYNC="rsync://mix8.clic.iap.fr/gentoo-portage"