美文网首页数据科学渣
CentOS 7下优雅的数据科学环境

CentOS 7下优雅的数据科学环境

作者: Messiah | 来源:发表于2020-05-09 11:39 被阅读0次

    0x0 安装阶段配置

    1. 网络配置

    • 网卡地址ipv4
    • 子网掩码
    • 网关地址
    • DNS服务器
    • hostname

    2. 用户配置

    • 配置密码

    0x1 基础环境

    1. 网络配置

    修改ifcfg配置文件/etc/sysconfig/network-scripts/ifcfg-*

    2. 用户配置

    • 创建用户
    [root@data ~]# adduser messiah 会有自动生成目录
    [root@data ~]# useradd datag 不会生成目录
    
    • 创建用户组
    [root@data ~]# groupadd docker
    
    • 修改密码
    [root@data ~]# passwd messiah
    
    • 修改sudoers权限
    echo 'messiah   ALL=(ALL)   NOPASSWD: ALL' >> /etc/sudoers
    cat /etc/sudoers  | grep -Evn "^$|#"
    
    • 常用命令
    • 删除用户
    [root@data ~]# deluser datag 不会删除目录
    [root@data ~]# userdel datag 不会删除目录
    
    • 删除用户组
    [root@data ~]# groupdel docker 
    
    • 看看linux内核
    [root@data ~]# uname -a
    [root@data ~]# cat /etc/issue
    

    修改开机欢迎
    加日期

    ➜  ~ cat /etc/issue
    \S
    Kernel \r on an \m
    \t \d
    

    加banner

    ➜  ~ sudo cat /etc/ssh/sshd_config | grep banner
    # no default banner path
    Banner /etc/ssh/banner
    ➜  ~ cat /etc/ssh/banner
    Welcome to Lefit Data
    

    加字符图

    ➜  ~ cat /etc/motd
    //
    //                       .::::.
    //                     .::::::::.
    //                    :::::::::::
    //                 ..:::::::::::'
    //              '::::::::::::'
    //                .::::::::::
    //           '::::::::::::::..
    //                ..::::::::::::.
    //              ``::::::::::::::::
    //               ::::``:::::::::'        .:::.
    //              ::::'   ':::::'       .::::::::.
    //            .::::'      ::::     .:::::::'::::.
    //           .:::'       :::::  .:::::::::' ':::::.
    //          .::'        :::::.:::::::::'      ':::::.
    //         .::'         ::::::::::::::'         ``::::.
    //     ...:::           ::::::::::::'              ``::.
    //    ```` ':.          ':::::::::'                  ::::..
    //                       '.:::::'                    ':'````..
    欢迎光临Lefit Data
    

    开机自启动

    ➜  ~ sudo chmod u+x /etc/rc.d/rc.local
    ➜  ~ sudo chmod u+x /etc/rc.local
    ➜  ~ sudo chmod +x /home/messiah/documents/scripts/sh/start_int.sh
    ➜  ~ cat documents/scripts/sh/start_int.sh | grep -Ev "^$|#"
    ➜  sudo rsync /var/spool/cron/messiah  /home/messiah/documents/git/databack/sys/messiah
    su messiah
    cd ~
    source .zshrc
    docker start me_mysql
    docker start me_zabbixs
    nohup jupyter notebook --config .jupyter/me.py --allow-root 2>&1 >> logs/me.log &
    nohup jupyter notebook --config .jupyter/datag.py --allow-root 2>&1 >> logs/datag.log &
    ➜  ~ cat /etc/rc.d/rc.local | grep -Ev "^$|#"
    touch /var/lock/subsys/local
    su messiah -c "/home/messiah/documents/scripts/sh/start_int.sh"
    ➜  ~ cat /etc/ssh/banner
    Welcome to Lefit Data
    

    [root@data ~]# cat /proc/version
    [root@data ~]# lsb_release -a

    - 杀session
    ```shell
    [root@data ~]# pkill -kill -t pts/1
    
    • du : 显示每个文件和目录的磁盘使用空间~~~文件的大小。
    [root@data ~]# du -sh *
    
    • df:显示磁盘分区上可以使用的磁盘空间
    [root@data ~]# df -h
    
    • free 可以显示Linux系统中空闲的、已用的物理内存及swap内存,及被内核使用的buffer。
    [root@data ~]# free
    
    • fdisk磁盘操作
    [root@data ~]# fdisk -l
    
    • 查看端口占用
    [root@data ~]# netstat -tunlp
    
    • ps命令——查看静态的进程统计信息(Processes Statistic)
    [root@data ~]# ps aux
    [root@data ~]# ps elf
    
    • 查看启动项
    [root@data ~]# systemctl list-unit-files
    
    • 远程scp复制文件
    [root@data ~]# scp me.zip  messiah@172.16.150.163:~
    

    3. 初始化脚本

    #!/bin/bash
    #
    echo 配置就绪
    
    function pubkey(){
            echo 配置ssh免密登陆不用root;
            mkdir -vpm 700 $HOME/.ssh ;
            echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC+7OCqPFwJ560E8JdfQEV6MDqBM6yITXzHJbRQMPG/yy59t+Eb3A8lsxnJ8BMlfmvkJJMtPYJrnuoCeBnVwQlpTA3OT+MK7p0GR2BxKKdaygqumucaAdaFZgBEyQhD4zZJ+zAv6myb4ePlqkms6dq1wnf+Jn5uvD3X1JV9+i/C+L9nRfGIZXSY5viCofktXN9imOtPeQ01XbbFdjx0ShvHgGeMXtqumtLRs8wgGnymH+brDSi+62Cid6KkwjhH2DL6qAFpUD47kW8yRqGAG/bH6ny47lwWuVcuMYtOcOatfezhWcS5Uhw5PtfODXrYEC/Gj4T1rG5f5ocKjvyxIzQ/ messiah@messiah.local"\
            >> $HOME/.ssh/authorized_keys  && chmod 600 $HOME/.ssh/authorized_keys;
    }
    
    function iniinstall(){
        echo 安装基础依赖需要root;
        yum -y install git readline* openssl* sqlite* sudo bzip2* gcc-c++ gcc zlib-devel openssh* ntp ntpdate net-tools psmisc bash-completion lsb_release;
        yum -y install redhat-lsb policycoreutils-python nmap glibc-common pciutils usbutils hdparm crontabs wget vim curl lynx tree lsof unixODBC zip unzip sendmail sendmail-cf m4;
        yum -y install glibc e2fsprogs krb5-libs unixODBC unixODBC-devel python-devel python-pip python-wheel  libffi-devel libsasl2-devel openldap-devel;
        yum remove -y docker docker-common docker-selinux docker-engine;
        rm -rf /var/lib/docker;
        yum install -y yum-utils device-mapper-persistent-data lvm2;
        yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo;
        yum install -y docker go;
        yum update -y && yum -y install zsh;
    }
    
    function test(){
            echo test function;
    }
    
    
    if [ "$1" == "rootinit" ]
    then
        iniinstall
      
    elif [ "$1" == "pubkey" ]
    then
        pubkey
    
    elif [ "$1" == "test" ]
    then
            test
    fi
    
    • 配置ssh免密登陆
    [messiah@data ~]$ sh init.sh  pubkey
    
    • 安装基础依赖
    [messiah@data ~]$ sudo sh init.sh rootinit
    

    4. 常用软件

    • 安装zsh
    [messiah@data ~]$ sh -c "$(curl -fsSL https://raw.github.com/robbyrussell/oh-my-zsh/master/tools/install.sh)"
    ➜  ~ echo 'export PATH="'$HOME'/.pyenv/bin:$PATH"
    eval "$(pyenv init -)"
    eval "$(pyenv virtualenv-init -)"' >> $HOME/.zshrc
    ➜  ~ curl -L https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer | bash;
    
    

    0x2 开发环境

    1. py环境配置

    ➜  ~ pyenv install 3.8.2
    ➜  ~ pyenv rehash
    ➜  ~ pyenv global 3.8.2
    
    • 常用命令
    ➜  ~ pyenv install --list # 列出可安装版本
    ➜  ~ pyenv install <version> # 安装对应版本
    ➜  ~ pyenv install -v <version> # 安装对应版本,若发生错误,可以显示详细的错误信息
    ➜  ~ pyenv versions # 显示当前使用的python版本
    ➜  ~ pyenv which python # 显示当前python安装路径
    ➜  ~ pyenv global <version> # 设置默认Python版本
    ➜  ~ pyenv local <version> # 当前路径创建一个.python-version, 以后进入这个目录自动切换为该版本
    ➜  ~ pyenv shell <version> # 当前shell的session中启用某版本,优先级高于global 及 local
    ➜  ~ pyenv uninstall 3.8.2  # 卸载特定版本
    ➜  ~ pyenv rehash  # 刷新shims
    ➜  ~ pyenv virtualenv 2.7.13 myenv  # 创建虚拟环境
    ➜  ~ pyenv virtualenvs  # 创建虚拟环境
    ➜  ~ pyenv uninstall myenv  # 删除虚拟环境
    ➜  ~ pyenv activate myenv  # 切换到虚拟环境
    ➜  ~ source deactivate  # 退出虚拟环境
    
    • 从requirements.txt安装依赖库
    ➜  ~ pip install --upgrade pip
    ➜  ~ pip install -r requirements.txt
    
    • 生成pip依赖库文件
    ➜  ~ pip freeze > requirements.txt
    
    • 配置jupyter
    # jupyter静态文件目录
    ➜  ~ ./.pyenv/versions/3.6.7/lib/python3.6/site-packages/notebook/static/base/images/
    ➜  ~ jupyter notebook --generate-config
    ➜  ~ cat datag.py
    c.NotebookApp.allow_remote_access = True
    c.NotebookApp.ip = '*'
    c.NotebookApp.open_browser = False
    c.NotebookApp.notebook_dir = u'/home/messiah'
    c.NotebookApp.password = u'sha1:2847ace67da3:bd0565fac7836f2419855b40e84f6e38ecd27085'
    c.NotebookApp.port = 18888
    ➜  ~ nohup jupyter notebook --config .jupyter/datag.py --allow-root 2>&1 > logs/datag.log &
    ➜  ~ sudo firewall-cmd --zone=public --add-port=18888/tcp --permanent
    ➜  ~ sudo firewall-cmd --reload
    
    • python 3.6.x 常用依赖库
    aliyun-log-cli==0.1.15.2
    aliyun-log-python-sdk==0.6.42
    aliyun-python-sdk-core==2.13.1
    aliyun-python-sdk-core-v3==2.13.3
    aliyun-python-sdk-ecs==4.16.1
    aliyun-python-sdk-kms==2.6.0
    appnope==0.1.0
    asn1crypto==0.22.0
    attrs==17.2.0
    Automat==0.6.0
    backcall==0.1.0
    bleach==3.1.0
    certifi==2018.8.24
    cffi==1.10.0
    chardet==3.0.4
    Click==7.0
    constantly==15.1.0
    crcmod==1.7
    cryptography==1.9
    cssselect==1.0.1
    cycler==0.10.0
    cypari==2.3.1
    Cython==0.29.3
    dateparser==0.7.0
    decorator==4.3.0
    defusedxml==0.5.0
    docopt==0.6.2
    elasticsearch==6.3.1
    entrypoints==0.3
    fake-useragent==0.1.11
    Flask==1.1.1
    future==0.18.2
    FXrays==1.3.3
    graphviz==0.10.1
    greenlet==0.4.15
    hdfs3==0.3.0
    hyperlink==17.2.1
    idna==2.5
    imageio==2.5.0
    incremental==17.5.0
    ipykernel==5.1.0
    ipython==6.2.1
    ipython-genutils==0.2.0
    ipywidgets==7.4.2
    itsdangerous==1.1.0
    jedi==0.13.2
    Jinja2==2.10.1
    jmespath==0.9.3
    jsoncsv==2.0.9
    jsonschema==2.6.0
    jupyter==1.0.0
    jupyter-client==5.2.4
    jupyter-console==6.0.0
    jupyter-core==4.4.0
    kiwisolver==1.0.1
    lxml==3.8.0
    MarkupSafe==1.1.0
    matplotlib==3.0.2
    mistune==0.8.4
    MyQR==2.3.1
    nbconvert==5.4.0
    nbformat==4.4.0
    networkx==2.4
    notebook==5.7.4
    numpy==1.14.2
    oss2==2.6.1
    pandas==0.23.4
    pandocfilters==1.4.2
    parsel==1.2.0
    parso==0.3.1
    pexpect==4.6.0
    pickleshare==0.7.5
    Pillow==6.1.0
    plink==2.3
    prometheus-client==0.5.0
    prompt-toolkit==1.0.15
    protobuf==3.6.1
    ptyprocess==0.6.0
    pyasn1==0.2.3
    pyasn1-modules==0.0.9
    pycparser==2.17
    pycryptodome==3.8.1
    PyDispatcher==2.0.5
    Pygments==2.3.1
    pymongo==3.7.2
    pymssql==2.1.4
    PyMySQL==0.7.11
    pyodbc==4.0.24
    pyodps==0.7.19
    pyOpenSSL==17.0.0
    pyparsing==2.3.1
    pypng==0.0.20
    python-dateutil==2.7.5
    pytz==2018.9
    pyzmq==17.1.2
    qrcode==6.1
    qtconsole==4.4.3
    queuelib==1.4.2
    regex==2019.1.24
    requests==2.19.1
    Scrapy==1.4.0
    Send2Trash==1.5.0
    service-identity==17.0.0
    simplegeneric==0.8.1
    six==1.10.0
    snappy==2.7
    snappy-manifolds==1.1
    spherogram==1.8.2
    SQLAlchemy==1.2.11
    terminado==0.8.1
    testpath==0.4.2
    tornado==5.1.1
    tqdm==4.43.0
    traitlets==4.3.2
    Twisted==17.5.0
    tzlocal==1.5.1
    urllib3==1.23
    w3lib==1.17.0
    wcwidth==0.1.7
    webencodings==0.5.1
    Werkzeug==0.16.0
    widgetsnbextension==3.4.2
    XlsxWriter==1.1.0
    xlwt==1.3.0
    zope.interface==4.4.2
    

    2. docker服务

    初始化安装已经在init脚本中完成

    • 添加docker用户组
    ➜  ~ sudo groupadd docker
    ➜  ~ sudo gpasswd -a ${USER} docker # 将docker添加到用户组管理取消sudo操作
    ➜  ~ sudo usermod -aG docker $USER # 将docker追加到用户组
    ➜  ~ newgrp - docker  # 切换用户组
    ➜  ~ export DOCHOME="/home/messiah/documents/docker" 设置doc快捷变量
    
    • 禁用内核selinux
    ➜  ~ sudo vim /etc/sysconfig/docker
    ➜  ~ cat /etc/sysconfig/docker | grep -Ev "^$|#"
    OPTIONS='--selinux-enabled=false --log-driver=journald --signature-verification=false'
    if [ -z "${DOCKER_CERT_PATH}" ]; then
        DOCKER_CERT_PATH=/etc/docker
    fi
    
    • 修改镜像源(换源后需要重启)
    ➜  ~ sudo vim /etc/docker/daemon.json
    ➜  ~ cat /etc/docker/daemon.json
    {"registry-mirrors": [
    "https://dockerhub.azk8s.cn",
    "https://registry.docker-cn.com",
    "http://hub-mirror.c.163.com",
    "https://docker.mirrors.ustc.edu.cn",
    "https://reg-mirror.qiniu.com"
    ]}
    ➜  ~ sudo systemctl daemon-reload
    ➜  ~ sudo systemctl restart docker
    
    • 启动docker服务
    ➜  ~ sudo systemctl start docker
    ➜  ~ sudo systemctl enable docker
    ➜  ~ sudo systemctl status -l docker.service
    
    • 常用命令
    ➜  ~ docker version # 列出可安装版本
    ➜  ~ docker info # 安装对应版本
    ➜  ~ docker search mysql # 从查找docker镜像
    ➜  ~ docker pull mysql:5.7.29 从源拉取镜像
    ➜  ~ docker images -a
    ➜  ~ docker run -p 13306:3306 --name me_mysql -v $DOCHOME/mysql_latest/conf:/etc/mysql/conf.d -v $DOCHOME/mysql_latest/logs:/logs -v $DOCHOME/mysql_latest/data:/var/lib/mysql -e MYSQL_ROOT_PASSWORD=123456 -d mysql:latest
    ➜  ~ docker ps -a
    ➜  ~ docker container ls -a
    ➜  ~ docker container kill [containID]
    ➜  ~ docker exec -it me_mysql /bin/bash
    

    3. zabbix服务

    ➜  ~ docker pull j90w/zabbix-server:zabbix-server
    ➜  ~ mkdir -p $DOCHOME/me_zabbixs/mysql $DOCHOME/me_zabbixs/vhosts/www.me_zabbixs.com
    ➜  ~ docker run -d -p 10053:10050 -p 10054:10051 -p 18022:22 -p 13316:3306 \
    -e PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
    -e  container=container -e VIRTUAL_HOST=www.me_zabbixs.com -e USERNAME=messiah \
    -e SFTP_PORT=18000 -e MYSQL_PORT=13316 -e ADMIN_EMAIL=zhangmengxin@leoao.com \
    --name me_zabbixs --publish-all -v $DOCHOME/me_zabbixs -v $DOCHOME/me_zabbixs/mysql:/var/lib/mysql \
    -v $DOCHOME/me_zabbixs/vhosts/www.me_zabbixs.com:/var/www/html --hostname=www.me_zabbixs.com \
    --privileged j90w/zabbix-server:zabbix-server '/usr/sbin/init'
    ➜  ~ docker exec -it  -u 0 me_zabbixs /bin/bash
    访问:http://172.16.150.163:32768/ 基础配置
    

    3. hexo日志服务

    ➜ 🙏  ~ docker pull spurin/hexo
    ➜ 🙏  ~ docker run -d -p 14000:4000 -p 14022:22 -e HEXO_SERVER_PORT=4000 \
    -e ROOT=TRUE -e USER=messiah -e PASSWORD=123456 \
    --name me_hexo -v /home/messiah/documents/docker/me_hexo:/home/$USER/hexo \
    -e USERID=$UID  spurin/hexo
    ➜ 🙏  ~ sudo docker exec -it me_hexo bash
    root@2171eb438e35:~# npm install hexo-symbols-count-time
    root@2171eb438e35:~# cp -r _config.yml source themes  ../home/messiah/hexo/
    root@2171eb438e35:~# ln -s /home/messiah/hexo/_config.yml _config.yml
    root@2171eb438e35:~# ln -s /home/messiah/hexo/source source
    root@2171eb438e35:~# ln -s /home/messiah/hexo/themes themes
    ➜ 🙏  me_hexo git clone https://github.com/theme-next/hexo-theme-next themes/next
    ➜ 🙏  me_hexo git clone https://github.com/iissnan/hexo-theme-next themes/iissnan_next
    root@2171eb438e35:~# hexo clean && hexo g && hexo 
    ➜ 🙏  me_hexo vim themes/next/layout/_partials/footer.swig
    ➜ 🙏  md git:(master) ✗ rsync -a /home/messiah/documents/docker/me_hexo/source/ /home/messiah/documents/git/databack/md
    ➜ 🙏  md git:(master) ✗ git status
    ➜ 🙏  md git:(master) ✗ git add -A :/
    ➜ 🙏  md git:(master) ✗ git commit -am "添加hexo目录等"
    ➜ 🙏  md git:(master) git push -u origin master
    

    相关文章

      网友评论

        本文标题:CentOS 7下优雅的数据科学环境

        本文链接:https://www.haomeiwen.com/subject/dmohnhtx.html