最近有个需求，需要把爬下来的数据写入到文件中，于是乎查了下相关的博客和文档，大概记录一下，方便以后自己用，这个基本上都是复制粘贴来的，主要是方便自己用，其他人看到了觉得有用就用，没用也可以吐槽，就是抄的反正。
Nodejs文件覆盖、追加、读取、全量复制
同步版 从这里复制的
逻辑处理类的实现：syncdata.js
const fs = require("fs");
/**
* update file
* @param res to update content
* @returns update operation result
*/
async function updateResFile (res) {
return new Promise((resolve, reject) => {
fs.writeFile(`${conf['workdir']}${conf['res']}`, res, (err) => {
if (err) {
console.error(`error happened when updateResFile inner: ${err.message}`);
reject(false);
return;
} else {
resolve(true);
return;
}
});
}).catch(error => {
console.error(`error happened when updateResFile: ${error.message}`);
return false;
});
}
/**
* break line append log
* @param log log content
* @returns append operation result
*/
async function appendFile (log) {
return new Promise((resolve, reject) => {
if ("windows" == conf['os']) {
log+='\r\n';
} else {
log+='\n';
}
fs.appendFile(`${conf['workdir']}${conf['log']}`, log, (err) => {
if (err) {
console.error(`error happened when appendFile inner: ${err.message}`);
reject(false);
return;
} else {
resolve(true);
return;
}
});
}).catch(error => {
console.error(`error happened when appendFile: ${error.message}`);
});
}
/**
* execute copy file logic
* @param {*} src
* @param {*} dst
* @param {*} callback
*/
async function handleDir (src, dst) {
return new Promise(async (resolve, reject) => {
fs.access(dst, fs.constants.F_OK, async (err) => {
if (err) {
fs.mkdir(dst, { recursive: true }, async (err) => {
if (err) {
console.error(`error happened when handleDir.mkdir: ${err.message}`);
reject(`error happened when handleDir.mkdir: ${err.message}`);
return;
} else {
await copyDir(src, dst);
}
});
} else {
await copyDir(src, dst);
}
resolve(true);
});
}).catch(error => {
console.error(`error happened when handleDir: ${error.message}`);
});
}
/**
* execute iterator copy directory
* @param src directory source path
* @param dst directory destination path
* @returns copy result
*/
async function copyDir (src, dst) {
return new Promise(async (resolve, reject) => {
fs.readdir(src, (err, paths) => {
paths.forEach(function(path){
var _src=src+'/'+path;
var _dst=dst+'/'+path;
fs.stat(_src, async function(err,stats){
if(err) {
console.error(`copyDir failed, caused by ${JSON.stringify(err)}`);
reject(false);
} else if(stats.isFile()){
let readable=fs.createReadStream(_src);
let writable=fs.createWriteStream(_dst);
readable.pipe(writable);
readable.on('end', () => {
resolve(true);
});
}else if(stats.isDirectory()){
await handleDir(_src, _dst, copyDir);
}
});
});
});
}).catch(error => {
console.error(`error happened when copyDir: ${error.message}`);
});
}
/**
* read file sync
* @param path filefullpath
* @returns file content
*/
async function readFile (path) {
return new Promise((resolve, reject) => {
fs.readFile(path, conf['encode'] , (err, data) => {
if (err) {
console.error(`readFile failed, caused by ${JSON.stringify(err)}`);
reject(false);
} else {
resolve(data);
return;
}
});
}).catch(error => {
console.error(`error happened when readFile: ${error.message}`);
});
}
/**
* main entrance
*/
async function syncFile () {
console.log(`begin ------- ${new Date().getTime()}`);
let cpRes = await copyDir('D:/开发文档/邮件文档/1-邮件附件', './2');
console.log(`result is ${cpRes}`);
console.log(`end ------- ${new Date().getTime()}`);
}
// call
syncFile ();
同一目录下的配置文件conf.json
{
"os" : "windows",
"encode" : "utf-8",
"workdir" : "/home/",
"latestVersion" : "latestVersion.txt",
"currentVersion" : "currentVersion.txt",
"log" : "log.txt",
"res" : "syncResult.txt",
"copySrc" : "/copySrc/",
"copyDst" : "/copyDst/",
"success-text" : "success",
"fail-text" : "fail",
"dateformat" : "YYYY-MM-DD"
}
执行方式：
cd到syncdata.js同一级目录，执行node syncdata.js
异步版 从这里复制的
所有方法都是异步的，如果需要同步则需要修改fs的方法，并使用Promise进行包装，另外需要注意，全部复制的方法需要传入两个目录，不带文件的path，并且是有最大文件打开数量限制的，如果是大文件但文件数量较少则适合使用，如果是小文件且数量大，则不适合，建议进行先打成压缩包，再进行传输。
逻辑处理类的实现：syncdata.js
const fs = require("fs");
// local path
const conf = require("./conf.json");
/**
* update file
* @param res to update content
* @returns update operation result
*/
function updateResFile (res) {
return fs.writeFileSync(`${conf['workdir']}${conf['res']}`, res);
}
/**
* break line append log
* @param log log content
* @returns append operation result
*/
function appendFile (log) {
if ("windows" == conf['os']) {
log+='\r\n';
} else {
log+='\n';
}
return fs.appendFileSync(`${conf['workdir']}${conf['log']}`, log);
}
/**
* handle directory
* @param {*} src
* @param {*} dst
* @param {*} callback
*/
function handleDir(src, dst) {
fs.access(dst, fs.constants.F_OK, (err) => {
if (err) {
fs.mkdirSync(dst);
copyDir(src, dst);
} else {
copyDir(src, dst);
}
});
}
/**
* execute iterator copy directory
* @param src directory source path
* @param dst directory destination path
* @returns copy result
*/
function copyDir (src, dst) {
// read file path sync
let paths = fs.readdirSync(src);
paths.forEach(function(path){
var _src=src+'/'+path;
var _dst=dst+'/'+path;
//stats object, include its own attribute
fs.stat(_src,function(err,stats){
if(err) {
console.log(`copyDir failed, caused by ${JSON.stringify(err)}`);
return false;
} else if(stats.isFile()){
let readable=fs.createReadStream(_src);
let writable=fs.createWriteStream(_dst);
readable.pipe(writable);
}else if(stats.isDirectory()){
handleDir(_src, _dst, copyDir);
}
});
});
return true;
}
/**
* read file sync
* @param path filefullpath
* @returns file content
*/
function readFile (path) {
return fs.readFileSync(path, conf['encode']);
}
/**
* main entrance
*/
function syncFile () {
// test code here
}
// call
syncFile ();
同一目录下的配置文件`conf.json`
```json
{
"os" : "windows",
"encode" : "utf-8",
"workdir" : "/home/",
"latestVersion" : "latestVersion.txt",
"currentVersion" : "currentVersion.txt",
"log" : "log.txt",
"res" : "syncResult.txt",
"copySrc" : "/copySrc/",
"copyDst" : "/copyDst/",
"success-text" : "success",
"fail-text" : "fail",
"dateformat" : "YYYY-MM-DD"
}
执行方式：
cd到syncdata.js同一级目录，执行node syncdata.js
网友评论