Svolence

Gitlab+DRBD+KEEPALIVED+NFS

Posted on  |    |   In   |     |   Visitors

Gitlab cluster deployment related services

1.Gitlab server

  • System:　ubuntu 14.04 x86_64
  • Gitlab master:　10.100.136.62　
  • Gitlab slave:　10.100.136.63　
  • Gitlab version:　GitLab Community Edition 8.13.5
  • Location:　/var/opt/gitlab

2.redis/pgsql

  • System:　centos 6.8 x86_64
  • Redis:　10.100.17.25:6380　
  • Pgsql:　10.100.17.25:5432　
  • Version:　redis:2.8，pgsql:9.6
  • Location:　/data/redis，/var/lib/pgsql/9.6

3.DRBD & keepalived & nfs

  • System:　centos 6.8 x86_64
  • DRBD master:　10.100.17.31　
  • DRBD slave:　10.100.17.32　
  • Version:　8.9.6
  • Location:
    • /usr/loca/etc/drbd
    • /keepalived
    • /etc/exports

4.gitlab cluser config

  • MASTER
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    # vi /etc/gitlab/gitlab.rb
    # Disable the built-in Postgres
    postgresql['enable'] = false
    # Fill in the values for database.yml
    gitlab_rails['db_adapter'] = "postgresql"
    gitlab_rails['db_encoding'] = "unicode"
    gitlab_rails['db_database'] = "gitlabhq_production"
    gitlab_rails['db_host'] = '10.100.17.25'
    gitlab_rails['db_port'] = '5432'
    gitlab_rails['db_username'] = 'gitlab'
    gitlab_rails['db_password'] = '123456'
    # Redis connection details
    redis['enable'] = false
    gitlab_rails['redis_port'] = '6380'
    gitlab_rails['redis_host'] = '10.100.17.25'
    #gitlab_rails['redis_password'] = '123456'
  • SLAVE
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    # vi /etc/gitlab/gitlab.rb
    # Disable the built-in Postgres
    postgresql['enable'] = false
    # Fill in the values for database.yml
    gitlab_rails['db_adapter'] = "postgresql"
    gitlab_rails['db_encoding'] = "unicode"
    gitlab_rails['db_database'] = "gitlabhq_production"
    gitlab_rails['db_host'] = '10.100.17.25'
    gitlab_rails['db_port'] = '5432'
    gitlab_rails['db_username'] = 'gitlab'
    gitlab_rails['db_password'] = '123456'
    # Redis connection details
    redis['enable'] = false
    gitlab_rails['redis_port'] = '6380'
    gitlab_rails['redis_host'] = '10.100.17.25'
    #gitlab_rails['redis_password'] = '123456'
    gitlab_shell['secret_token'] = '4f008660152b2b649352ddf41f1d2d8098f98b1f5c23a0cb48a5ffbb3f065a9d8a11f8c7f70459f08a028d2048bab1ad3b370fcda732e315ed2b08a6ae707c87'
    gitlab_rails['otp_key_base'] = '89d90e3f7a9cc257f6ec8f88b78d0f072c5795154398387dbb5cbee90c1ccded855d8df03f1d5ea5b5dbec488f13bd02643b9c44c084331aa0184cf80a527f3d'
    gitlab_rails['secret_key_base'] = '370fb2996bbef6b13e11694f50d25b2cd2d3213697fe53afee5e6b154e364a9ab52f5f78a701903a858f975b6c51eb541f8d97bd0996813d30ab9ec0d0858fba'
    gitlab_rails['db_key_base'] = '7c58a519b10b167496364100f0cc12f188cd8823aaef915656626f29a7132defda3135c220fef449ae4d7dd3e49780de4b33dde77eda05855ef843a58467126d'
1
touch /etc/gitlab/skip-auto-migrations #skip auto migrations
  • COMMAND
    1
    2
    3
    sudo gitlab-ctl status/stop/restart
    sudo gitlab-ctl tail # view log

5.keepalived & DRBD config

  • MASTER

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    ! Configuration File for keepalived
    global_defs {
       notification_email {
         heyatao@imooc.com
       }
       notification_email_from op@imooc.com
       smtp_server 127.0.0.1
       smtp_connect_timeout 30
       router_id nfs_drbd0
       vrrp_skip_check_adv_addr
       vrrp_strict
       vrrp_garp_interval 0
       vrrp_gna_interval 0
    }
    vrrp_script check_nfs {
        script "/keepalived/script/check_nfs.sh"
        interval 2
        weight -60
    }
    vrrp_instance nfs_drbd0 {
        state MASTER
        interface eth1
        virtual_router_id 222
        priority 100
        advert_int 1
        authentication {
            auth_type PASS
            auth_pass Nfs_DrBd
        }
        
        track_script {
            check_nfs
        }
        virtual_ipaddress {
            10.100.17.9
        }
       
      notify_stop "/keepalived/script/nfs_stop.sh"
      notify_master "/keepalived/script/change_drbd_to_primary.sh"
      notify_backup "/keepalived/script/change_drbd_to_secondary.sh"
    }

  • SLAVE

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    ! Configuration File for keepalived
    global_defs {
       notification_email {
         heyatao@imooc.com
       }
       notification_email_from op@imooc.com
       smtp_server 127.0.0.1
       smtp_connect_timeout 30
       router_id nfs_drbd0
       vrrp_skip_check_adv_addr
       vrrp_strict
       vrrp_garp_interval 0
       vrrp_gna_interval 0
    }
    vrrp_script check_nfs {
        script "/keepalived/script/check_nfs.sh"
        interval 2
        weight -60
    }
    vrrp_instance nfs_drbd0 {
        state BACKUP
        interface eth2
        virtual_router_id 222
        priority 50
        advert_int 1
        authentication {
            auth_type PASS
            auth_pass Nfs_DrBd
        }
        
        track_script {
            check_nfs
        }
        virtual_ipaddress {
            10.100.17.9
        }
       
      notify_stop "/keepalived/script/nfs_stop.sh"
      notify_master "/keepalived/script/change_drbd_to_primary.sh"
      notify_backup "/keepalived/script/change_drbd_to_secondary.sh"
    }

master priority > slave priority
one or more vip address，virtual_router_id should be different

  • NFS FAILOVER

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    #!/bin/bash
    log_name='check_nfs'
    log_dir='/keepalived/log'
    nfs_flag="nfsd"
    VIP="10.100.17.9"
    flag_exists=$(ps -ef | grep "${nfs_flag}" | grep -v grep | wc -l)
    drbd_role=$(drbdsetup status |head -1 | awk -F':' '{print $2}')
    time=` date "+%F  %H:%M:%S"`
    echo -e "----------${time}  check nfs start ------------\n" >> ${log_dir}/${log_name}.log
    if [ "${flag_exists}" -eq 0 ];
    then
        # Try to reboot once
        echo -e "----------${time}  restart nfs ------------\n" >> ${log_dir}/${log_name}.log
        /sbin/service nfs restart &>> ${log_dir}/${log_name}.log
        flag_exists=$(ps -ef | grep "${nfs_flag}" | grep -v grep | wc -l)
        if [ "${flag_exists}" -eq 0 ];
        then
            if [ "${drbd_role}" == "Primary" ];
            then
               echo -e "----------${time}  ${log_name} ------------\n" >> ${log_dir}/${log_name}.log
               cd /keepalived/script/ && bash change_drbd_to_secondary.sh &>> ${log_dir}/${log_name}.log
               cd /keepalived && bash load_keepalived.sh stop   &>> ${log_dir}/${log_name}.log
               echo -e "----------The End--------------------------\n" >> ${log_dir}/${log_name}.log
            fi
        fi
    fi
    echo -e "----------${time}  check nfs end ------------\n" >> ${log_dir}/${log_name}.log

  • COMMAND

    1
    2
    3
    4
    5
    6
    7
    drbdadm status drbd_disk0
    drbdadm up/down drbd_disk0
    drbdadm --force primary drbd_disk0
    service nfs restart/start/stop

  • Manual sovled split brain

    • drdbadm disconnect resource
    • drbdadm secondary resource

    • drbdadm connect --discard-my-data resource

    • On the other node (the split brain survivor), if its connection state is also StandAlone, you would enter:

    • drbdadm disconnect resource

    • drbdadm connect resource

6.NFS mount

  • Server

    1
    /nfs_disk0 10.100.*(rw,no_root_squash)

  • Client

    1
    10.100.17.9:/nfs_disk0/gitlab/git-data/repositories /var/opt/gitlab/git-data/repositories nfs4 defaults 0 0