[Qemu OpenChannelSSD] Bad Block

Intro

BBT是一个lun上所有block状态的列表，每个block可能的状态如下：
一个BBT的项目数(entry number)是geo->nplanes * geo-nblocks.

另外，Bad-Block Table是需要由开发者自己维护的，可以通过mark&&flush来修改在内存中的BBT或持久化BBT（将内存中的BBT写入设备：写到哪里可能还需要去探索qemu中相关的处理）.

/**
 * Representation of valid values of bad-block-table states
 */
enum nvm_bbt_state {
    NVM_BBT_FREE = 0x0, ///< Block is free AKA good
    NVM_BBT_BAD = 0x1,  ///< Block is bad
    NVM_BBT_GBAD = 0x2, ///< Block has grown bad
    NVM_BBT_DMRK = 0x4, ///< Block has been marked by device side
    NVM_BBT_HMRK = 0x8  ///< Block has been marked by host side
};

struct nvm_dev {
    char name[NVM_DEV_NAME_LEN];    ///< Device name e.g. "nvme0n1"
    char path[NVM_DEV_PATH_LEN];    ///< Device path e.g. "/dev/nvme0n1"
    struct nvm_addr_fmt fmt;    ///< Device address format
    struct nvm_addr_fmt_mask mask;  ///< Device address format mask
    struct nvm_geo geo;     ///< Device geometry
    uint64_t ssw;           ///< Bit-width for LBA fmt conversion
    int pmode;          ///< Default plane-mode I/O
    int fd;             ///< Device fd / IOCTL handle
    int erase_naddrs_max;       ///< Maximum # of address for erase
    int read_naddrs_max;        ///< Maximum # of address for read
    int write_naddrs_max;       ///< Maximum # of address for write
    int bbts_cached;        ///< Whether to cache bbts
    size_t nbbts;           ///< Number of entries in cache
    struct nvm_bbt **bbts;      ///< Cache of bad-block-tables
    enum meta_mode meta_mode;   ///< Flag to indicate the how meta is w
};

• 从dev_open可以看出：

1. 一个lun一个bbt.
2. 初始时每个lun对应的bbt（dev->bbts[i]）都是空（NULL）.

struct nvm_dev *nvm_dev_open(const char *dev_path)
{
...
    dev->bbts_cached = 0;
    dev->nbbts = dev->geo.nchannels * dev->geo.nluns;
    dev->bbts = malloc(sizeof(*dev->bbts) * dev->nbbts);
    for (size_t i = 0; i < dev->nbbts; ++i)
        dev->bbts[i] = NULL;

    return dev;
}

• 使用bbt的时候如果没有设置标记nvm_dev_set_bbts_cached(dev, 1)可能会出现一些预想不到的结果，具体原因正在分析中.(可提issue)

• 使用bbt设置时（nvm_bbt_mark其最大addrs也是不能超过NVM_NADDR_MAX）

• test code:

#include <liblightnvm.h>
#include <malloc.h>
#include <nvm.h>
#include <stdio.h>
#include <string.h>

static char nvm_dev_path[NVM_DEV_PATH_LEN] = "/dev/nvme0n1";
static struct nvm_dev *dev;
static const struct nvm_geo *geo;
static struct nvm_addr lun_addr;

static int channel = 0;
static int lun = 0;
static void *Buf_for_1_read = NULL; //Buf for reading 1 sector
static void *Buf_for_1_meta_read = NULL; // Buf for reading 1 metadata

const int FirstBLK = 0;
const int SecondBLK = 1;
const int Plane = 0;

const uint16_t State_HBAD = NVM_BBT_HMRK;
const uint16_t State_FREE = NVM_BBT_FREE;

const char* TestStr = "K:Foo;V:Bar";

#include "common.h"


int setup(void)
{
    dev = nvm_dev_open(nvm_dev_path);
    if (!dev) {
        perror("nvm_dev_open");
        return -1;
    }
    geo = nvm_dev_get_geo(dev);
    lun_addr.ppa = 0;
    lun_addr.g.ch = channel;
    lun_addr.g.lun = lun;

    Buf_for_1_read =  nvm_buf_alloc(geo, 2 * geo->sector_nbytes);
    if (!Buf_for_1_read) {
        FAIL_ALLOC;
        return -1;
    }
    
    Buf_for_1_meta_read = nvm_buf_alloc(geo, 2 * geo->meta_nbytes);
    if (!Buf_for_1_meta_read) {
        FAIL_ALLOC;
        return -1;
    }

    return 0;
}

int teardown(void)
{
    nvm_dev_close(dev);
    if ( Buf_for_1_read ) {
        free(Buf_for_1_read);
    }
    if ( Buf_for_1_meta_read) {
        free(Buf_for_1_meta_read);
    }
    return 0;
}

static inline int _bbt_idx(const struct nvm_dev *dev,
               const struct nvm_addr addr)
{
    return addr.g.ch * dev->geo.nluns + addr.g.lun;
}

uint64_t alignblk(struct nvm_addr addr)
{
    struct nvm_addr alg;
    alg.ppa = addr.ppa;
    alg.g.pg = 0;
    alg.g.sec = 0;
    return alg.ppa;
}

int GetPmode(int npl)
{
    int x;
    switch (npl) {
    case 1: 
        x = NVM_FLAG_PMODE_SNGL; break;
    case 2:
        x = NVM_FLAG_PMODE_DUAL; break;
    case 4:
        x = NVM_FLAG_PMODE_QUAD; break;
    default:
        x = -1;
    }
    return x;
}

void EraseNpl_1Blk(struct nvm_addr wh)//
{
    struct nvm_ret ret;
    ssize_t res;
    
    const int npl = geo->nplanes;
    int pmode = GetPmode(npl);
    struct nvm_addr whichblk[npl];
    for(int i = 0; i < npl; ++i){
        whichblk[i].ppa = alignblk(wh);
        whichblk[i].g.pl = i;
    }

    res = nvm_addr_erase(dev, whichblk, npl, pmode, &ret);//Erase 1 block of all planes inside a lun.
    if(res < 0){
        FAIL_ERASE;
        nvm_ret_pr(&ret);
    }
}
void Write_1Sector(struct nvm_addr wh, const char *IO, int use_meta, const char *Meta)
{
    struct nvm_ret ret;
    ssize_t res;
    int pmode = NVM_FLAG_PMODE_SNGL;
    void *bufptr = NULL, *bufptr_meta = NULL;

    int addr_valid = nvm_addr_check(wh, geo);
    if(addr_valid){
        ADDR_INVALID;
        nvm_bounds_pr(addr_valid);
        goto OUT;
    }

    //buf setup
    bufptr = nvm_buf_alloc(geo, geo->sector_nbytes);//sector size
    if(!bufptr){
        FAIL_ALLOC;
        goto OUT;
    }
    memcpy(bufptr, IO, strlen(IO));

    if(use_meta){
        bufptr_meta = nvm_buf_alloc(geo, geo->meta_nbytes);
        if(!bufptr_meta){
            FAIL_ALLOC;
            goto FREE_OUT1;
        }
        memcpy(bufptr_meta, Meta, geo->meta_nbytes);
    }


    //2. write
    res = nvm_addr_write(dev, &wh, 1, bufptr, 
        use_meta ? bufptr_meta : NULL, pmode, &ret);//Write 1 sector
    if(res < 0){
        FAIL_WRITE;
    }

    free(bufptr_meta);
FREE_OUT1:
    free(bufptr);
OUT:
    if(res < 0){
        nvm_ret_pr(&ret);
    }
    return;
}
void Read_1Sector(struct nvm_addr wh, int use_meta)
{
    struct nvm_ret ret;
    ssize_t res;

    int pmode = NVM_FLAG_PMODE_SNGL;

    res = nvm_addr_read(dev, &wh, 1, Buf_for_1_read,
        use_meta ? Buf_for_1_meta_read : NULL, pmode, &ret);//only use the first sector_nbytes
    if(res < 0){
        FAIL_READ;
        nvm_ret_pr(&ret);
    }
}


void Erase1LunALLBlk(struct nvm_addr lun_addr)//
{
//#define TEST_EraseAllLunBlk_FUNC
#define WAY1 1
#define WAY2 2
#define WAY WAY2        //WAY1 is not good!!
    struct nvm_ret ret;
    ssize_t res;
    int i, j;
    const int npl = geo->nplanes;
    const int nblks = geo->nblocks;
    int pmode = GetPmode(npl);
//    int pmode = NVM_FLAG_PMODE_SNGL;
    if (pmode < 0) {
        PLANE_EINVAL;
        return;
    }
    
    const int max_naddrs_1_time = NVM_NADDR_MAX;
    const int naddrs_1_time = max_naddrs_1_time / npl;

    struct nvm_addr whichblks[NVM_NADDR_MAX];//for all block
   
    for(i = 0; i < NVM_NADDR_MAX; ++i){
        whichblks[i].ppa = 0;
        whichblks[i].g.ch = lun_addr.g.ch; 
        whichblks[i].g.lun = lun_addr.g.lun;
    }
    printf("Nblks: %d, Npl: %d\n", nblks, npl);
    int t = 0;
    int erased = 0;
    while (erased < nblks) {
        int issue_nblks = NVM_MIN(nblks - erased, naddrs_1_time);
        int naddrs = issue_nblks * npl;
        //SUM: issue_nblks * npl
        
        if (WAY == WAY1) {      //TEST Result: 2 ways of addr layout are both OK.
            //Way 1:
            for (i = 0; i < issue_nblks; ++i) {
                for (j = 0; j < npl; j++) {
                    whichblks[j * issue_nblks + i].g.blk = erased + i;
                    whichblks[j * issue_nblks + i].g.pl = j;
                }
            }
        }else{
            //Way 2:
            for (i = 0; i < issue_nblks; ++i) {
                for (j = 0; j < npl; j++) {
                    whichblks[i * npl + j].g.blk = erased + i;
                    whichblks[i * npl + j].g.pl = j;
                }
            }
        }

        
#ifdef TEST_EraseAllLunBlk_FUNC
        char title[10];
        sprintf(title, "idx%d", t);
        My_pr_addr_cap(title);

        for (int idx = 0; idx < naddrs; ++idx) {
            My_pr_nvm_addr(whichblks[idx]);
        }
#endif
        res = nvm_addr_erase(dev, whichblks, naddrs, pmode, &ret);//Erase SUM blocks
        if(res < 0){
            FAIL_ERASE;
            printf("%d naadrs From:\n", naddrs);
            My_pr_addr_with_str("Bad Addr", whichblks[0]);
            nvm_ret_pr(&ret);
            printf("Stop\n");
            return ;
        }

        erased += naddrs_1_time;
        t++;
    }
}

void Set1LunALLFree(struct nvm_addr lun_addr)
{    
    struct nvm_ret ret;
    int res;
    int i, j;
    const int npl = geo->nplanes;
    const int nblks = geo->nblocks;

    const int max_naddrs_1_time = NVM_NADDR_MAX;
    const int naddrs_1_time = max_naddrs_1_time / npl;
   

    struct nvm_addr whichblks[NVM_NADDR_MAX];//for all block
    for(i = 0; i < NVM_NADDR_MAX; ++i){
        whichblks[i].ppa = 0;
        whichblks[i].g.ch = lun_addr.g.ch; 
        whichblks[i].g.lun = lun_addr.g.lun;
    }
    printf("Nblks: %d, Npl: %d\n", nblks, npl);
    int t = 0;
    int erased = 0;
    while (erased < nblks) {
        int issue_nblks = NVM_MIN(nblks - erased, naddrs_1_time);
        int naddrs = issue_nblks * npl;
        //SUM: issue_nblks * npl

        if (WAY == WAY1) {      //TEST Result: 2 ways of addr layout are both OK.
            //Way 1:
            for (i = 0; i < issue_nblks; ++i) {
                for (j = 0; j < npl; j++) {
                    whichblks[j * issue_nblks + i].g.blk = erased + i;
                    whichblks[j * issue_nblks + i].g.pl = j;
                }
            }
        }else{
            //Way 2:
            for (i = 0; i < issue_nblks; ++i) {
                for (j = 0; j < npl; j++) {
                    whichblks[i * npl + j].g.blk = erased + i;
                    whichblks[i * npl + j].g.pl = j;
                }
            }
        }

        res = nvm_bbt_mark(dev, whichblks, naddrs, State_FREE, &ret);
        if (res < 0) {
            MARK_BBT_FAIL;
            printf("%d naadrs From:\n", naddrs);
            My_pr_addr_with_str("Bad Addr", whichblks[0]);
            nvm_ret_pr(&ret);
            printf("Stop\n");
            return ;
        }

        erased += naddrs_1_time;
        t++;
    }
}




void test_get_bbt_1(void)
{
    struct nvm_ret ret = {};
    const struct nvm_bbt *bbt;
    bbt = nvm_bbt_get(dev, lun_addr, &ret);
    if (!bbt) {
        GET_BBT_FAIL;
        nvm_ret_pr(&ret);
    }
    My_nvm_bbt_pr(bbt);
}

void test_run_erase_blk_1_lun(void)
{
    Erase1LunALLBlk(lun_addr);
}

void test_write_to_blk(int plnum, int blknum)//write all pages inside a blk of block=blknum, plane=plnum
{
    const int npl = geo->nplanes;
    struct nvm_addr a0;
    a0.ppa = lun_addr.ppa;
    a0.g.pl = plnum;
    a0.g.blk = blknum;

    for (int i = 0; i < geo->npages; i++) {
        a0.g.pg = i;
        Write_1Sector(a0, TestStr, 0, NULL);
        ((char*)Buf_for_1_read)[0] = '\0';
        Read_1Sector(a0, 0);
        if (strcmp((char*)Buf_for_1_read, TestStr) != 0) {
            NOT_THE_SAME_IO;
            My_pr_addr_with_str("not same:", a0);
        }
    }
}

void Test_Erase1()
{
    struct nvm_ret ret;
    ssize_t res;
    int pl[4] = {0,1,0,1};
    int blk[4] = {0,0,1,1};
    struct nvm_addr a0[4];
    for (int i = 0; i < 4; i++) {
        a0[i].ppa = 0;
        a0[i].g.pl = pl[i];
        a0[i].g.blk = blk[i];
    }
    res = nvm_addr_erase(dev, a0, 4, NVM_FLAG_PMODE_DUAL, &ret);//Erase 4 blocks
    if(res < 0){
        FAIL_ERASE;
        nvm_ret_pr(&ret);
    }
}

void Test_Erase2()
{
    struct nvm_ret ret;
    ssize_t res;
    int pl[4] = {0,0,1,1};
    int blk[4] = {0,1,0,1};
    struct nvm_addr a0[4];
    for (int i = 0; i < 4; i++) {
        a0[i].ppa = 0;
        a0[i].g.pl = pl[i];
        a0[i].g.blk = blk[i];
    }
    res = nvm_addr_erase(dev, a0, 4, NVM_FLAG_PMODE_DUAL, &ret);//Erase 4 blocks
    if(res < 0){
        FAIL_ERASE;
        nvm_ret_pr(&ret);
    }
}
void test_E1_ok()
{
    struct nvm_addr a0;
    a0.ppa = 0;
    a0.g.blk = 1;
    Test_Erase1();
    Write_1Sector(a0,"This is the test.", 0, NULL);
}
void test_E2_fail()
{
    struct nvm_addr a0;
    a0.ppa = 0;
    a0.g.blk = 1;
    Test_Erase2();
    Write_1Sector(a0,"This is the test.", 0, NULL);
}
void test_set_cache_bbt(void)
{
    nvm_dev_set_bbts_cached(dev, 1);
}

void test_clean_all_mark_all_free(void)
{
    struct nvm_ret ret;
    ssize_t res;
    Erase1LunALLBlk(lun_addr);
    Set1LunALLFree(lun_addr);
    if(nvm_bbt_flush(dev, lun_addr, &ret)){//flush the lun's bbt
        FLUSH_BBT_FAIL;
        nvm_ret_pr(&ret);
    }
}

void test_write_into_1st_2ed_blk(void)//write something into all pages inside the 1st/2ed block of all plane inside <lun_addr>
{
    for (int i = 0; i < geo->nplanes; i++) {
        test_write_to_blk(i, FirstBLK);
        test_write_to_blk(i, SecondBLK); 
    }
}

void set_1st_2ed_blk_flush(uint16_t flag)
{
    struct nvm_ret ret;
    int res;
    const int nblks = 2;//1st 2ed
    const int naddrs = nblks * geo->nplanes;//the 1st/2ed block of LUN 0 across all plane
    struct nvm_addr addrs[naddrs];
    //*****Sequence: Must Be Like WAY2*****
    for (int i = 0; i < geo->nplanes; i++) {
        for (int j = 0; j < nblks; j++) {
            addrs[j * geo->nplanes + i].ppa = lun_addr.ppa;
            addrs[j * geo->nplanes + i].g.pl = i;
            addrs[j * geo->nplanes + i].g.blk = j;
        }
    }
    //My_pr_naddrs_with_str("Test", addrs, naddrs);
    
    res = nvm_bbt_mark(dev, addrs, naddrs, flag, &ret); 
    if (res < 0) {
       MARK_BBT_FAIL;
       nvm_ret_pr(&ret);
    }
    if(nvm_bbt_flush(dev, lun_addr,&ret)){//flush the lun's bbt
        FLUSH_BBT_FAIL;
        nvm_ret_pr(&ret);
    }
}


void test_set_1st_2ed_blk_bad(void)//set them as HBAD block so that background GC will then GC them.
{
    set_1st_2ed_blk_flush(State_HBAD);
}



void test_GC(void)//**find all HBAD blocks and erase them**(erase operation must be issued to all plane inside a LUN)
{
    struct nvm_ret ret = {};
    const struct nvm_bbt *bbt;
    int nbad = 0;
    ssize_t res;
    bbt = nvm_bbt_get(dev, lun_addr, &ret);
    if (!bbt) {
        GET_BBT_FAIL;
        nvm_ret_pr(&ret);
    }

    const int max_naddrs_1_time = NVM_NADDR_MAX;
    struct nvm_addr whichblks[NVM_NADDR_MAX];//for all block
    int idx = 0;


    //iterator all block, from nvm_bbt_pr
    for (int i = 0; i < bbt->nblks; i += bbt->dev->geo.nplanes) {
        int blk = i / bbt->dev->geo.nplanes;
        int blk_num = blk;
        int pl_num = 0;
        for (int blk = i; blk < (i+ bbt->dev->geo.nplanes); ++blk, ++pl_num) {
            if(bbt->blks[blk] == State_HBAD){
                printf("HBAD: %d %d\n", pl_num, blk_num);

                //setup address array
                whichblks[idx].ppa = lun_addr.ppa;
                whichblks[idx].g.pl = pl_num;
                whichblks[idx].g.blk = blk_num;
                idx++;

                nbad++;
            }
        }
        
        if (max_naddrs_1_time - idx < bbt->dev->geo.nplanes) {
            //do erase.
            
            res = nvm_addr_erase(dev, whichblks, idx, NVM_FLAG_PMODE_DUAL, &ret);
            if(res < 0){
                FAIL_ERASE;
                nvm_ret_pr(&ret);
            }
            
            //set them as FREE again
            if (nvm_bbt_mark(dev, whichblks, idx, State_FREE, &ret) < 0) {
               MARK_BBT_FAIL;
               nvm_ret_pr(&ret);
            }
            if(nvm_bbt_flush(dev, lun_addr,&ret)){//flush the lun's bbt
                FLUSH_BBT_FAIL;
                nvm_ret_pr(&ret);
            } 
            
            idx = 0;
        }

    }
    if (idx > 0) {
        //do erase.
        
        res = nvm_addr_erase(dev, whichblks, idx, NVM_FLAG_PMODE_DUAL, &ret);
        if(res < 0){
            FAIL_ERASE;
            nvm_ret_pr(&ret);
        }
        
        //set them as FREE again
        if (nvm_bbt_mark(dev, whichblks, idx, State_FREE, &ret) < 0) {
           MARK_BBT_FAIL;
           nvm_ret_pr(&ret);
        }
        if(nvm_bbt_flush(dev, lun_addr,&ret)){//flush the lun's bbt
            FLUSH_BBT_FAIL;
            nvm_ret_pr(&ret);
        } 
        
        idx = 0;
    }
    printf("HBAD block number: %d\n", nbad);
 }
 typedef void (* FuncType) (void);
void RunTests()
{
    FuncType tests[] = { 
//        test_run_erase_blk_1_lun,        
//        test_write_to_1st_blk,
//        test_write_to_2ed_blk,
//        test_get_bbt_1,
//        test_E1_ok,
//        test_E2_fail
        test_set_cache_bbt,     
        test_clean_all_mark_all_free,
        test_get_bbt_1,
        test_write_into_1st_2ed_blk,
        test_set_1st_2ed_blk_bad,
        test_get_bbt_1,
        test_GC,
        test_get_bbt_1
        };
    const char *teststr[] = {
//        "test_run_erase_blk_1_lun",        
//        "test_write_to_1st_blk",
//        "test_write_to_2ed_blk",
//        "test_get_bbt_1",
//        "test_E1_ok",
//        "test_E2_fail"
        "test_set_cache_bbt",
        "test_clean_all_mark_all_free",
        "test_get_bbt_1",
        "test_write_into_1st_2ed_blk",
        "test_set_1st_2ed_blk_bad",
        "test_get_bbt_1",
        "test_GC",
        "test_get_bbt_1"
        };
    for(int i = 0; i < (sizeof(tests) / sizeof(FuncType)); i++){
        printf("====Test %d : %s====\n", i, teststr[i]);
        tests[i]();
    }
}
int main()
{
    if( setup() < 0){
        goto OUT;
    }
    RunTests();

OUT:
    teardown();
    return 0;
}

common.h:
#ifndef COMMON_H
#define COMMON_H
#include <liblightnvm.h>
#include <nvm.h>

#define DEBUG_MSG(MSG) do{ printf("[DEBUG] - "#MSG"\n"); }while(0)

//for io_issue:
#define FAIL_ERASE DEBUG_MSG(FAIL_ERASE)
#define FAIL_ALLOC DEBUG_MSG(FAIL_ALLOC)
#define FAIL_WRITE DEBUG_MSG(FAIL_WRITE)
#define FAIL_READ  DEBUG_MSG(FAIL_READ)
#define ADDR_INVALID DEBUG_MSG(ADDR_INVALID)
#define THE_SAME_IO DEBUG_MSG(THE_SAME_IO) 
#define NOT_THE_SAME_IO DEBUG_MSG(NOT_THE_SAME_IO)
#define THE_SAME_META DEBUG_MSG(THE_SAME_META)
#define NOT_THE_SAME_META DEBUG_MSG(NOT_THE_SAME_META)

//for bbt_issue:
#define PLANE_EINVAL DEBUG_MSG(PLANE_EINVAL)
#define GET_BBT_FAIL DEBUG_MSG(GET_BBT_FAIL)
#define FLUSH_BBT_FAIL DEBUG_MSG(FLUSH_BBT_FAIL)
#define MARK_BBT_FAIL DEBUG_MSG(MARK_BBT_FAIL)
//for common use:
#define NVM_MIN(x, y) ({                \
        __typeof__(x) _min1 = (x);      \
        __typeof__(y) _min2 = (y);      \
        (void) (&_min1 == &_min2);      \
        _min1 < _min2 ? _min1 : _min2; })


inline static void My_pr_addr_cap(const char* str)
{
#define digestlen 8
    char digest[digestlen];
    strncpy(digest, str, digestlen);
    digest[digestlen - 1] = '\0';

    printf("%8s | %s | %s | %s | %-4s | %3s | %s \n",
        digest,"ch", "lun", "pl", "blk", "pg", "sec");
}
inline static void My_pr_nvm_addr(struct nvm_addr addr)
{
    printf("         | %2d | %3d | %2d | %4d | %3d | %d\n",
           addr.g.ch, addr.g.lun, addr.g.pl,
           addr.g.blk, addr.g.pg, addr.g.sec);
}

inline static void My_pr_addr_with_str(const char *str, struct nvm_addr x)
{
    My_pr_addr_cap(str);
    My_pr_nvm_addr(x);
}
inline static void My_pr_naddrs_with_str(const char *str, struct nvm_addr x[], int naddrs)
{
    My_pr_addr_cap(str);
    for (int i = 0; i < naddrs; i++) {
        My_pr_nvm_addr(x[i]);
    }
}

inline static void My_nvm_bbt_pr(const struct nvm_bbt *bbt)
{
    int nnotfree = 0;
    const int Pr_num = 4;
    int pred = 0, pr_sr = 0;
    if (!bbt) {
        printf("bbt { NULL }\n");
        return;
    }

    printf("bbt {\n");
    printf("  addr"); nvm_addr_pr(bbt->addr);
    printf("  nblks(%lu) {", bbt->nblks);
    for (int i = 0; i < bbt->nblks; i += bbt->dev->geo.nplanes) {
        int blk = i / bbt->dev->geo.nplanes;
        if (pred < Pr_num/*first Pr_num ones*/ 
            || i == bbt->nblks - bbt->dev->geo.nplanes/*last one*/) {
            printf("\n    blk(%04d): [ ", blk);
            for (int blk = i; blk < (i + bbt->dev->geo.nplanes); ++blk) {
                nvm_bbt_state_pr(bbt->blks[blk]);
                printf(" ");
                if (bbt->blks[blk]) {
                    ++nnotfree;
                }
            }
            printf("]");
            pred++;
        }else if(!pr_sr){
            printf("\n....");
            pr_sr = 1;
        }


    }
    printf("\n  }\n");
    printf("  #notfree(%d)\n", nnotfree);
    printf("}\n");
}


#endif

1.设置cache，
2.擦除整个lun
3.拿bbt
4.写入lun_0的每个plane的前2个block（一共4个block在我的环境geo下）
5.设置这4个block为HBAD
5.1拿bbt看是否有变化
6.模拟一个GC过程，把设为HBAD的block擦除掉
7.拿bbt
Output:

====Test 0 : test_set_cache_bbt====
====Test 1 : test_clean_all_mark_all_free====
Nblks: 2044, Npl: 2
Nblks: 2044, Npl: 2
====Test 2 : test_get_bbt_1====
bbt {
  addr(0x0000000000000000){ ch(00), lun(00), pl(0), blk(0000), pg(000), sec(0) }
  nblks(4088) {
    blk(0000): [ FREE(0) FREE(0) ]
    blk(0001): [ FREE(0) FREE(0) ]
    blk(0002): [ FREE(0) FREE(0) ]
    blk(0003): [ FREE(0) FREE(0) ]
....
    blk(2043): [ FREE(0) FREE(0) ]
  }
  #notfree(0)
}
====Test 3 : test_write_into_1st_2ed_blk====
====Test 4 : test_set_1st_2ed_blk_bad====
====Test 5 : test_get_bbt_1====
bbt {
  addr(0x0000000000000000){ ch(00), lun(00), pl(0), blk(0000), pg(000), sec(0) }
  nblks(4088) {
    blk(0000): [ HBAD(8) HBAD(8) ]
    blk(0001): [ HBAD(8) HBAD(8) ]
    blk(0002): [ FREE(0) FREE(0) ]
    blk(0003): [ FREE(0) FREE(0) ]
....
    blk(2043): [ FREE(0) FREE(0) ]
  }
  #notfree(4)
}
====Test 6 : test_GC====
HBAD: 0 0
HBAD: 1 0
HBAD: 0 1
HBAD: 1 1
HBAD block number: 4
====Test 7 : test_get_bbt_1====
bbt {
  addr(0x0000000000000000){ ch(00), lun(00), pl(0), blk(0000), pg(000), sec(0) }
  nblks(4088) {
    blk(0000): [ FREE(0) FREE(0) ]
    blk(0001): [ FREE(0) FREE(0) ]
    blk(0002): [ FREE(0) FREE(0) ]
    blk(0003): [ FREE(0) FREE(0) ]
....
    blk(2043): [ FREE(0) FREE(0) ]
  }
  #notfree(0)
}

总结

1. 一个bbt对应一个lun
2. bbt是由用户自己维护的，并且可以持久化到设备中.(设备中对应逻辑需要到qemu中查看)
3. 使用bbt时需要设置dev->bbts_cached标记