day18(MongoDB)

安装MongoDB

yum install libcurl openssl -y
mkdir /opt/mongo_cluster/ -p
#mkdir /data/soft -p
cd /data/soft/
#wget https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-3.6.13.tgz
tar zxvf mongodb-linux-x86_64-3.6.13.tgz -C /opt/mongo_cluster/
cd /opt/mongo_cluster/ && ln -s mongodb-linux-x86_64-3.6.13 mongodb
mkdir /opt/mongo_cluster/mongo_27017/{conf,logs,pid} -p
mkdir /data/mongo_cluster/mongo_27017 -p

配置启动：

1.修改配置文件
cat > /opt/mongo_cluster/mongo_27017/conf/mongodb.conf  << EOF
systemLog:
  destination: file   
  logAppend: true  
  path: /opt/mongo_cluster/mongo_27017/logs/mongodb.log

storage:
  journal:
    enabled: true
  dbPath: /data/mongo_cluster/mongo_27017
  directoryPerDB: true
  wiredTiger:
     engineConfig:
        cacheSizeGB: 1
        directoryForIndexes: true
     collectionConfig:
        blockCompressor: zlib
     indexConfig:
        prefixCompression: true

processManagement:
  fork: true
  pidFilePath: /opt/mongo_cluster/mongo_27017/pid/mongod.pid

net:
  port: 27017
  bindIp: 127.0.0.1,10.0.0.51
EOF
2.启动命令
cd /opt/mongo_cluster/
mongodb/bin/mongod -f mongo_27017/conf/mongodb.conf 
3.检查
ps -ef|grep mongo
netstat -lntup|grep 27017
4.登录
mongodb/bin/mongo db01:27017
5.关闭
mongodb/bin/mongod -f mongo_27017/conf/mongodb.conf --shutdown

优化警告

#解决大内存页警告
echo never > /sys/kernel/mm/transparent_hugepage/enabled 
echo never > /sys/kernel/mm/transparent_hugepage/defrag

vim /etc/init.d/disable-transparent-hugepages
#!/bin/bash
### BEGIN INIT INFO
# Provides:          disable-transparent-hugepages
# Required-Start:    $local_fs
# Required-Stop:
# X-Start-Before:    mongod mongodb-mms-automation-agent
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: Disable Linux transparent huge pages
# Description:       Disable Linux transparent huge pages, to improve
#                    database performance.
### END INIT INFO

case $1 in
  start)
    if [ -d /sys/kernel/mm/transparent_hugepage ]; then
      thp_path=/sys/kernel/mm/transparent_hugepage
    elif [ -d /sys/kernel/mm/redhat_transparent_hugepage ]; then
      thp_path=/sys/kernel/mm/redhat_transparent_hugepage
    else
      return 0
    fi

    echo 'never' > ${thp_path}/enabled
    echo 'never' > ${thp_path}/defrag

    re='^[0-1]+$'
    if [[ $(cat ${thp_path}/khugepaged/defrag) =~ $re ]]
    then
      # RHEL 7
      echo 0  > ${thp_path}/khugepaged/defrag
    else
      # RHEL 6
      echo 'no' > ${thp_path}/khugepaged/defrag
    fi

    unset re
    unset thp_path
    ;;
esac

chmod 755 /etc/init.d/disable-transparent-hugepages     
chkconfig --add disable-transparent-hugepages

#优化rlimits警告
vim /etc/profile
ulimit -u 64000
source /etc/profile
重启mongo服务


#普通用户启动
/opt/mongo_cluster/mongodb/bin/mongod -f /opt/mongo_cluster/mongo_27017/conf/mongodb.conf --shutdown
useradd mongo
echo '123456'|passwd --stdin mongo
chown -R mongo:mongo /opt/mongo_cluster
chown -R mongo:mongo /data/mongo_cluster
su - mongo
vim .bashrc 
export PATH=/opt/mongo_cluster/mongodb/bin:$PATH
source .bashrc
mongod -f /opt/mongo_cluster/mongo_27017/conf/mongodb.conf
mongo

插入数据

db.test.insert({"name":"zhangya","age":27,"ad":"北京市朝阳区"})
db.test.insert({"name":"zhangya","age":27,"ad":"北京市朝阳区"})
db.test.insert({"name":"yazhang","age":28,"ad":"北京市朝阳区"})
db.test.insert({"name":"xiaozhang","age":28,"ad":"北京市朝阳区"})
db.test.insert({"name":"xiaozhang","age":28,"ad":"北京市朝阳区","sex":"boy"})


批量插入多条数据
db.inventory.insertMany( [
    { "item": "journal", "qty": 25, "size": { "h": 14, "w": 21, "uom": "cm" }, "status": "A" },
    { "item": "notebook", "qty": 50, "size": { "h": 8.5, "w": 11, "uom": "in" }, "status": "A" },
    { "item": "paper", "qty": 100, "size": { "h": 8.5, "w": 11, "uom": "in" }, "status": "D" },
    { "item": "planner", "qty": 75, "size": { "h": 22.85, "w": 30, "uom": "cm" }, "status": "D" },
    { "item": "postcard", "qty": 45, "size": { "h": 10, "w": 15.25, "uom": "cm" }, "status": "A" }
 ]);

查询数据

db.test.find()
db.test.findOne()
db.inventory.find()
db.inventory.findOne()


条件查询
db.inventory.find("status":"D")
db.inventory.find({"size.uom":"cm"})
db.inventory.find({"size.uom":"cm","qty":{$lt:50}})


db.inventory.find({"status":"A",$or:[{"qty":{$lt:30},{"item":"/^p/"}]})

更新数据

更新一条
db.inventory.updateOne(
    { "item" : "paper" }, 
    {
      $set: {  "size.uom" : "cm",  "status" : "P" },
      $currentDate: { "lastModified": true }
    }
)


更新多条
db.inventory.updateMany(
    { "qty" : { $lt: 50 } },
    {
       $set: { "size.uom" : "cm", "status": "P" },
       $currentDate : { "lastModified": true }
    }
)

删除数据

db.inventory.deleteOne({"status":"D"})
db.inventory.deleteMany({"status":"p"})

mongostat

mongotop

用户授权认证

use admin
db.createUser(
{
user: "admin",
pwd: "123456",
roles:[ { role: "root", db:"admin"}]}
)

修改配置文件
vim /opt/mongo_cluster/mongo_27017/conf/mongodb.conf
security:
  authorization: enabled

重启服务
mongod -f /opt/mongo_cluster/mongo_27017/conf/mongodb.conf --shutdown
mongod -f /opt/mongo_cluster/mongo_27017/conf/mongodb.conf

登录
mongo
show dbs 

mongo -uadmin -p 
show dbs

权限与角色

1.使用admin用户登录 
mongo -uadmin -p 
show dbs 

2.切换到test库，并创建一个用户赋予不同库的不同角色
use test
db.createUser(
  {
    user: "myTester",
    pwd: "xyz123",
    roles: [ { role: "readWrite", db: "test" },
             { role: "read", db: "test2" } ]
  }
)

3.在test库下插入测试数据 
db.test.insert({"name":"zhangya","age":27,"ad":"北京市朝阳区"})
db.test.insert({"name":"zhangya","age":27,"ad":"北京市朝阳区"})
db.test.insert({"name":"yazhang","age":28,"ad":"北京市朝阳区"})
db.test.insert({"name":"xiaozhang","age":28,"ad":"北京市朝阳区"})

4.在test2库下插入测试数据
use test2
db.test2.insert({"name":"zhangya","age":27,"ad":"北京市昌平区"})
db.test2.insert({"name":"zhangya","age":27,"ad":"北京市昌平区"})
db.test2.insert({"name":"yazhang","age":28,"ad":"北京市昌平区"})

5.使用myTester用户登录到test验证库
mongo -umyTester -pxyz123 db01:27017/test
show dbs 
db 

6.切换到test库，测试能否读写
use test
db.test.insert({"name":"58NB","age":27,"ad":"北京市朝阳区"})
db.test.find()

7.切换到test2库，测试能否读写
use test2
db.test2.insert({"name":"58NB","age":27,"ad":"北京市昌平区"})
db.test2.find()

主从复制

副本集配置
1.创建配置文件目录
mkdir -p /opt/mongo_cluster/mongo_2801{7,8,9}/{conf,logs,pid}


2.修改副本集配置文件
cat >/opt/mongo_cluster/mongo_28017/conf/mongo_28017.conf <<EOF
systemLog:
  destination: file   
  logAppend: true  
  path: /opt/mongo_cluster/mongo_28017/logs/mongodb.log

storage:
  journal:
    enabled: true
  dbPath: /data/mongo_cluster/mongo_28017
  directoryPerDB: true
  wiredTiger:
     engineConfig:
        cacheSizeGB: 0.5 
        directoryForIndexes: true
     collectionConfig:
        blockCompressor: zlib
     indexConfig:
        prefixCompression: true

processManagement:
  fork: true
  pidFilePath: /opt/mongo_cluster/mongo_28017/pid/mongod.pid

net:
  port: 28017
  bindIp: 127.0.0.1,10.0.0.51

replication:
   oplogSizeMB: 1024 
   replSetName: dba58 
EOF

3.复制配置文件到其他目录
cp /opt/mongo_cluster/mongo_28017/conf/mongo_28017.conf /opt/mongo_cluster/mongo_28018/conf/mongo_28018.conf
cp /opt/mongo_cluster/mongo_28017/conf/mongo_28017.conf /opt/mongo_cluster/mongo_28019/conf/mongo_28019.conf

4.修改配置文件里的端口
sed -i 's#28017#28018#g' /opt/mongo_cluster/mongo_28018/conf/mongo_28018.conf  
sed -i 's#28017#28019#g' /opt/mongo_cluster/mongo_28019/conf/mongo_28019.conf

5.创建数据目录
mkdir /data/mongo_cluster/mongo_2801{7,8,9}

6.启动多实例
mongod -f /opt/mongo_cluster/mongo_28017/conf/mongo_28017.conf 
mongod -f /opt/mongo_cluster/mongo_28018/conf/mongo_28018.conf 
mongod -f /opt/mongo_cluster/mongo_28019/conf/mongo_28019.conf


zhangya 2019/7/8 17:43:28
7.初始化副本集
config = {
_id : "dba58", 
members : [
{_id : 0, host : "db01:28017"},
{_id : 1, host : "db01:28018"},
{_id : 2, host : "db01:28019"},
] }
rs.initiate(config) 

8.查看副本集状态
rs.status()

9.从库无权限，配置文件添加（家目录下隐藏文件），
vim .mongorc.js 
rs.slaveOk()

权重调整+主库降级

1.查看配置信息
rs.conf()
2.配置权重
config = rs.conf()
config.members[2].priority=100
3.重新导入配置信息
rs.reconfig(config)
4.主库降级，发起重新选举 
rs.stepDown()
5.恢复成默认权重
config = rs.conf()
config.members[2].priority=1
rs.reconfig(config) 
rs.stepDown()

增加节点

1.安装配置启动新节点
mkdir /opt/mongo_cluster/mongo_28010/{conf,logs,pid} -p
mkdir /data/mongo_cluster/mongo_28010
cp /opt/mongo_cluster/mongo_28017/conf/mongo_28017.conf /opt/mongo_cluster/mongo_28010/conf/mongo_28010.conf
sed -i 's#28017#28010#g' /opt/mongo_cluster/mongo_28010/conf/mongo_28010.conf
mongod -f /opt/mongo_cluster/mongo_28010/conf/mongo_28010.conf
mongo db01:28010

2.主库执行添加新节点操作
use admin
rs.add("db01:28010")

删除旧节点

1.主库操作，删除旧节点
mongo db01:28017
rs.remove("db01:28010")
2.节点下线
mongod -f /opt/mongo_cluster/mongo_28010/conf/mongo_28010.conf --shutdown

添加仲裁节点

1.清空28010节点的数据
rm -rf /data/mongo_cluster/mongo_28010/*
2.重新启动
mongod -f /opt/mongo_cluster/mongo_28010/conf/mongo_28010.conf
3.检查
mongo db01:28010
4.主库操作
mongo db01:28017
rs.addArb("db01:28010")
5.查看节点状态
mongo db01:28010

备份恢复

导出
mongoexport --port 28017 -d dba -c inventory -o /backup/log.json
mongoexport --port 28017 -d dba -c inventory --type=csv -f item,qty,size,status  -o /backup/log.csv
导入
mongoimport --port 28017 -d oldboy -c inventory /backup/log.json

导入CSV
mongoimport  --port 28017 -d dba58 -c log2 --type=csv --headerline -f item,qty,size,status  --file  /backup/log.csv


mysql导出csv
select * from world.city into outfile '/tmp/city1.csv' fields terminated by ',';

导入csv
mongoimport --port 28017  -d world  -c city --type=csv -f ID,Name,CountryCode,District,Population --file  /tmp/city1.csv

备份恢复

全备
mongodump  --port 28017 -o /data/backup

只备一个库
mongodump  --port 28017 -d world -o /data/backup

恢复：
mongorestore --port 28017 -d world1  /data/backup/world/

恢复之前先删除
mongorestore --port 28017 -d world1  /data/backup/world/  --drop

备份恢复故障案列

背景：每天 0 点全备，oplog 恢复窗口为 48 小时 某天，上午 10 点 world.city 业务表被误删除。 恢复思路： 
0、停应用 
2、找测试库 
3、恢复昨天晚上全备 
4、截取全备之后到 world.city 误删除时间点的 oplog，并恢复到测试库 
5、将误删除表导出，恢复到生产库 
恢复步骤： 模拟故障环境： 1、全备数据库 
模拟原始数据 
[mongo@db01 ~]$ mongo db01:28017 
dba58:PRIMARY> use wo 
dba58:PRIMARY> for(var i = 1 ;i < 20; i++) { 
    db.ci.insert({a: i}); 
} 

2.全备 
[mongo@db01 ~]$ rm -rf /data/backup/* 
[mongo@db01 ~]$ mongodump --port 28017 --oplog -o /data/backup 
--oplog 功能:在备份同时,将备份过程中产生的日志进行备份 
文件必须存放在/data/backup 下,自动命令为 oplog.bson 
3.再次模拟数据 
[mongo@db01 ~]$ mongo db01:28017 
dba58:PRIMARY> use wo 
dba58:PRIMARY> db.ci1.insert({id:1}) 
dba58:PRIMARY> db.ci2.insert({id:2}) 
dba58:PRIMARY> show tables 
ci 
ci1 
ci2 
4.上午 10 点：删除 wo 库下的 ci 表 10:00 时刻,误删除 
dba58:PRIMARY> db.ci.drop() 
true 
dba58:PRIMARY> show tables 
ci1 
ci2 
5.备份现有的 oplog.rs 表 
[mongo@db01 ~]$ mongodump --port 28017 -d local -c oplog.rs  -o /data/backup 
6.截取 oplog 并恢复到 drop 之前的位置 更合理的方法：登陆到原数据库 
[mongo@db01 ~]$ mongo db01:28017   
dba58:PRIMARY> use local 
dba58:PRIMARY> db.oplog.rs.find({ns:"wo.$cmd"}).pretty(); 
{ 
        "ts" : Timestamp(1562510676, 1), 
        "t" : NumberLong(20), 
        "h" : NumberLong("-6833353909054714034"), 
        "v" : 2, 
        "op" : "c", 
        "ns" : "wo.$cmd", 
        "ui" : UUID("327876dd-5032-4914-a080-58707e0cd867"), 
        "wall" : ISODate("2019-07-07T14:44:36.031Z"), 
        "o" : { 
                "drop" : "ci" 
        } 
} 
7.恢复备份+应用 oplog 
[mongo@db01 ~]$ cd /data/backup/local/ 
[mongo@db01 /data/backup/local]$ ls 
oplog.rs.bson  oplog.rs.metadata.json 
[mongo@db01 /data/backup/local]$ cp oplog.rs.bson ../oplog.bson 
[mongo@db01 /data/backup/local]$ rm -rf /data/backup/local/ 
[mongo@db01 ~]$ mongorestore --port 28017 --oplogReplay --oplogLimit "1562510676:1"  --drop  
/data/backup/ 
[mongo@db01 ~]$ mongo db01:28017   
dba58:PRIMARY> use wo 
switched to db wo 
dba58:PRIMARY> show tables 
ci 
ci1 
ci2