美文网首页
2020-03-17 想了解文件内存映射mmap的过来看看啊

2020-03-17 想了解文件内存映射mmap的过来看看啊

作者: 懂你的 | 来源:发表于2020-03-17 15:55 被阅读0次

测试fread,fwrite,mmap_read,mmap_write分别在多线程单线程下的效率。

结论是mmap效率没有任何提高。

fread最好是单线程，一个线程读多个文件速度快很多。

#include

#include

#include

#include

#include

#include

#include

#include

#include

#include

#include

using namespace std;

static int operator-(struct timeval& lsh, struct timeval& rsh)

{

if (lsh.tv_sec==rsh.tv_sec)

{

return lsh.tv_usec - rsh.tv_usec;

    }

else

    {

return (lsh.tv_sec-rsh.tv_sec)*1000000 + (lsh.tv_usec - rsh.tv_usec);

    }

}

void mmap_read_thread(int idx)

{

struct timeval start;

    struct timeval mid;

    struct timeval end;

    long read_len =1024 *1024 *20;

    char *pReaderData =new char[read_len +1]; //200MB

    gettimeofday(&start, NULL);

    std::string file_name ="/media/ubuntu/harddisk/data_old/t/y/" + to_string(idx +1)+ ".txt";

    int fd = open(file_name.c_str(), O_RDONLY,00777);

    long file_length = lseek(fd, 0, SEEK_END);

    u_char * p_map = (u_char *)mmap(NULL, file_length, PROT_READ, MAP_PRIVATE, fd, 0);

    close(fd);

    int times = (file_length + read_len -1) / read_len;

    long last_len = file_length % read_len;

    long all_length =0;

    for(int y =0; y < times; y++)

{

if(all_length + last_len >= file_length)

read_len = last_len;

        long pass_length = y * read_len;

        memcpy(pReaderData, (u_char *)p_map + pass_length, read_len);

        printf("file = %d, times = %d, read_len = %d\n", idx, y, read_len);

        all_length += read_len;

    }

munmap(p_map,file_length);

    gettimeofday(&end, NULL);

    struct tm stTime;

    char strTemp[128];

    localtime_r(&end.tv_sec, &stTime);

    strftime(strTemp, sizeof(strTemp)-1, "%Y-%m-%d %H:%M:%S", &stTime);

    printf("start=%s.d\n", strTemp, end.tv_usec);

    printf("mmpread_spend_time = %d\n", (end-start)/1000000);

    delete[] pReaderData;

    pReaderData =NULL;

}

void fread_thread(int idx)

{

struct timeval start;

    struct timeval end;

    struct tm stTime;

    char strTemp[40];

    int read_len =1024 *1024 *200;

    char *pReaderData =new char[read_len];

    gettimeofday(&start, NULL);

    localtime_r(&start.tv_sec, &stTime);

    strftime(strTemp, sizeof(strTemp)-1, "%Y-%m-%d %H:%M:%S", &stTime);

    printf("start=%s.d\n", strTemp, start.tv_usec);

    string file_name ="/media/ubuntu/harddisk/data_old/t/y/" + to_string(idx +1)+ ".txt";

    FILE *reader = fopen(file_name.c_str(),"rb");

    fseek(reader, 0, SEEK_END);

    long file_length = ftell(reader);

    fseek(reader, 0, SEEK_SET);

    int times = (file_length + read_len -1) / read_len;

    long last_len = file_length % read_len;

    long all_length =0;

    for(int i =0;i

{

if(all_length + last_len >= file_length)

read_len = last_len;

        int real_len = fread(pReaderData, 1, read_len,reader);

        all_length += real_len;

    }

fclose(reader);

    gettimeofday(&end, NULL);

    localtime_r(&end.tv_sec, &stTime);

    strftime(strTemp, sizeof(strTemp)-1, "%Y-%m-%d %H:%M:%S", &stTime);

    printf("start=%s.d\n", strTemp, end.tv_usec);

    printf("fwrite_spend_time = %d\n", (end-start)/1000000);

}

void fwrite_thread(int file_idx)

{

struct timeval start;

    struct timeval mid;

    struct timeval end;

    int read_len =1024 *1024 *200;

    int file_length =1024 *1024 *1024;

    char *pReaderData =new char[read_len]; //200MB

    FILE *reader = fopen("/media/ubuntu/harddisk/data_old/t/0_104646.mp4","rb");

    int real_len = fread(pReaderData, 1, read_len,reader);

    fclose(reader);

    gettimeofday(&start, NULL);

    gettimeofday(&mid, NULL);

    for(int i =1; i <9; i++)

{

std::string file_name ="/media/ubuntu/harddisk/data_old/t/x/" + to_string(file_idx)+ "_" + to_string(i);

        FILE *pfile = fopen(file_name.c_str(), "wb");

        for(int j =0;j<5;j++)

fwrite(pReaderData, 1, read_len, pfile);

        fclose(pfile);

        printf("file write finished ! %d_%d\n", file_idx, i);

    }

gettimeofday(&end, NULL);

    delete[] pReaderData;

    pReaderData =NULL;

}

void mmap_write_thread(int file_idx)

{

struct timeval start;

    struct timeval mid;

    struct timeval end;

    int read_len =1024 *1024 *200;

    int file_length =1024 *1024 *1024;

    char *pReaderData =new char[read_len]; //200MB

    FILE *reader = fopen("/media/ubuntu/harddisk/data_old/t/0_104646.mp4","rb");

    int real_len = fread(pReaderData, 1, read_len,reader);

    fclose(reader);

    gettimeofday(&start, NULL);

    void *pFile[25];

    for(int i =1; i <9 ; i++)

{

std::string file_name ="/media/ubuntu/harddisk/data_old/t/x/" + to_string(file_idx)+ "_" + to_string(i);

        int fd = open(file_name.c_str(), O_RDWR |O_CREAT );

        if (fd<0)

{

printf("open error!\n");

return;

        }

lseek(fd, file_length-1, SEEK_SET);

        write(fd, "", 1);

        pFile[i] = mmap(NULL, file_length, PROT_WRITE, MAP_SHARED, fd, 0);

        if (MAP_FAILED == pFile)

{

perror("mmap");

return;

        }

close(fd);

        fd = -1;

    }

printf("file create finished ! %d_%d\n", file_idx);

    gettimeofday(&mid, NULL);

    for(int i =1; i <9 ; i++)

{

int write_len =0;

        for(int j =0;j<5;j++)

{

memcpy((unsigned  char*)pFile[i] + write_len, pReaderData, real_len);

            write_len += real_len;

        }

if (-1==munmap(pFile[i], file_length))

{

perror("munmap");

        }

printf("file write finished ! %d_%d\n", file_idx, i);

    }

gettimeofday(&end, NULL);

    struct tm stTime;

    char strTemp[128];

    localtime_r(&end.tv_sec, &stTime);

    strftime(strTemp, sizeof(strTemp)-1, "%Y-%m-%d %H:%M:%S", &stTime);

    printf("start=%s.d\n", strTemp, end.tv_usec);

    printf("fwrite_spend_time = %d\n", (end-start)/1000000);

    delete[] pReaderData;

    pReaderData =NULL;

}

void test_fwrite()

{

struct timeval start;

    struct timeval end;

    struct tm stTime;

    char strTemp[40];

    // 346,1529=25.5,290,297

    int thread_number =3;

    gettimeofday(&start, NULL);

    localtime_r(&start.tv_sec, &stTime);

    strftime(strTemp, sizeof(strTemp)-1, "%Y-%m-%d %H:%M:%S", &stTime);

    printf("start=%s.d\n", strTemp, start.tv_usec);

    std::thread *read_threads =new std::thread[3];

    for(int i =0; i < thread_number; ++i){

read_threads[i]= std::thread(&fwrite_thread, i);

    }

for(int i =0; i < thread_number; ++i){

read_threads[i].join();

    }

delete [] read_threads;

    gettimeofday(&end, NULL);

    localtime_r(&end.tv_sec, &stTime);

    strftime(strTemp, sizeof(strTemp)-1, "%Y-%m-%d %H:%M:%S", &stTime);

    printf("start=%s.d\n", strTemp, end.tv_usec);

    printf("fwrite_spend_time = %d\n", (end-start)/1000000);

}

void test_fread()

{

struct timeval start;

    struct timeval end;

    struct tm stTime;

    char strTemp[40];

    // 346,1529=25.5,290,297

    int thread_number =3;

    gettimeofday(&start, NULL);

    localtime_r(&start.tv_sec, &stTime);

    strftime(strTemp, sizeof(strTemp)-1, "%Y-%m-%d %H:%M:%S", &stTime);

    printf("start=%s.d\n", strTemp, start.tv_usec);

    std::thread *read_threads =new std::thread[3];

    for(int i =0; i < thread_number; ++i){

read_threads[i]= std::thread(&fread_thread, i);

    }

for(int i =0; i < thread_number; ++i){

read_threads[i].join();

    }

delete [] read_threads;

    gettimeofday(&end, NULL);

    localtime_r(&end.tv_sec, &stTime);

    strftime(strTemp, sizeof(strTemp)-1, "%Y-%m-%d %H:%M:%S", &stTime);

    printf("start=%s.d\n", strTemp, end.tv_usec);

    printf("fwrite_spend_time = %d\n", (end-start)/1000000);

}

void test_mmap_read()

{

struct timeval start;

    struct timeval end;

    struct tm stTime;

    char strTemp[40];

    // 346,1529=25.5,290,297

    int thread_number =3;

    gettimeofday(&start, NULL);

    localtime_r(&start.tv_sec, &stTime);

    strftime(strTemp, sizeof(strTemp)-1, "%Y-%m-%d %H:%M:%S", &stTime);

    printf("start=%s.d\n", strTemp, start.tv_usec);

    std::thread *read_threads =new std::thread[3];

    for(int i =0; i < thread_number; ++i){

read_threads[i]= std::thread(&mmap_read_thread, i);

    }

for(int i =0; i < thread_number; ++i){

read_threads[i].join();

    }

delete [] read_threads;

    gettimeofday(&end, NULL);

    localtime_r(&end.tv_sec, &stTime);

    strftime(strTemp, sizeof(strTemp)-1, "%Y-%m-%d %H:%M:%S", &stTime);

    printf("start=%s.d\n", strTemp, end.tv_usec);

    printf("fwrite_spend_time = %d\n", (end-start)/1000000);

}

void test_mmap_write()

{

struct timeval start;

    struct timeval end;

    struct tm stTime;

    char strTemp[40];

    // 5 min,1570=26

    int thread_number =3;

    gettimeofday(&start, NULL);

    localtime_r(&start.tv_sec, &stTime);

    strftime(strTemp, sizeof(strTemp)-1, "%Y-%m-%d %H:%M:%S", &stTime);

    printf("start=%s.d\n", strTemp, start.tv_usec);

    std::thread *read_threads =new std::thread[3];

    for(int i =0; i < thread_number; ++i){

read_threads[i]= std::thread(&mmap_write_thread, i);

    }

for(int i =0; i < thread_number; ++i){

read_threads[i].join();

    }

delete [] read_threads;

    gettimeofday(&end, NULL);

    localtime_r(&end.tv_sec, &stTime);

    strftime(strTemp, sizeof(strTemp)-1, "%Y-%m-%d %H:%M:%S", &stTime);

    printf("start=%s.d\n", strTemp, end.tv_usec);

    printf("mmap_spend_time = %d\n", (end-start)/1000000);

}

int main()

{

//    mmap_read_thread(0);//86

//    mmap_read_thread(1);

//    mmap_read_thread(2);

//    test_mmap_read();//3:52:49.d,13:39:03.d

//14min

//    fread_thread(0);

//    fread_thread(1);

//    fread_thread(2);

//    test_fread();

// 346,326

//    test_fwrite();

//    fwrite_thread(0);

//    fwrite_thread(1);//105S

//    fwrite_thread(2);

// 5 min,518

//    test_mmap_write();

//    mmap_write_thread(0);

//    mmap_write_thread(1);

//    mmap_write_thread(2);

//mmap 83.525s

//fwrite 86

//    test2();

    return 1;

}

相关文章

  • 2020-03-17 想了解文件内存映射mmap的过来看看啊

    测试fread,fwrite,mmap_read,mmap_write分别在多线程单线程下的效率。 结论是mmap...

  • APP优化 —— MMAP内存映射

    mmap 一种内存映射文件的方法 mmap将一个文件或者其它对象映射进内存。文件被映射到多个页上，如果文件的大小不...

  • 内存映射

    mmap 背景 mmap将一个文件或者其它对象映射进内存，文件以mmap的方式映射到用户的虚拟内存空间，省去了从内...

  • 关于mmap不为人知的秘密

    mmap初入 我们常说的mmap，其实是一种内存映射文件的方法，mmap将一个文件或者其它对象映射进内存。但是更加...

  • mmap

    简单的目录 mmap基础概念 mmap内存映射原理 mmap和常规文件操作的区别 mmap优点总结 mmap相关函...

  • linux手册翻译——mmap(2)

    mmap，munmap-将文件或设备映射（消取映射）到内存 mmap() mmap()将在调用者进程的虚拟地址空间...

  • mmap和MMKV

    一. mmap 1. 什么是mmap mmap是一种内存映射文件的方法，即将一个文件或者其他对象映射到进程的地址...

  • 25用户态内存映射

    mmap 将虚拟地址空间映射到物理内存 匿名映射，即虚拟内存和物理内存直接映射文件映射，将文件映射到虚拟内存ima...

  • 是时候了解下 mmap 了

    1、mmap基础概念 mmap 是一种内存映射文件的方法，即将一个文件或者其他对象映射到进程的地址空间，实现文件磁...

  • 【转】linux库函数mmap()原理

    linux库函数mmap()原理 目录 1.mmap基本概念 2.mmap内存映射原理 3.mmap和常规文件操作...

网友评论

      本文标题：2020-03-17 想了解文件内存映射mmap的过来看看啊

      本文链接：https://www.haomeiwen.com/subject/bfbtyhtx.html

      延伸阅读

      深度阅读

      • 您也可以注册成为美文阅读网的作者，发表您的原创作品、分享您的心情！

      栏目导航

      热点阅读

      关于我们|服务条款|联系我们|2020-03-17 想了解文件内存映射mmap的过来看看啊|投稿指南|网站地图|RSS订阅|排版工具|手机版

      提供经典美文摘抄,优美散文欣赏,现代诗歌精选,短篇小说,心情随笔,表白情书范文,故事会在线阅读欣赏

      Copyright © 2014-2023 Haomeiwen.com All Rights Reserved. 好美文阅读网 版权所有

      备案信息：桂公网安备 45052102000051号 · 桂ICP备13007215号-3

      本站所收录作品、热点评论等信息部分来源互联网，目的只是为了系统归纳学习和传递资讯

      所有作品版权归原创作者所有，与本站立场无关，如不慎侵犯了你的权益，请联系我们告知，我们将做删除处理！