通过图片的链接(如:https://img.haomeiwen.com/i11722017/e806b092026a0502.png?imageMogr2/auto-orient/strip%7CimageView2/2/w/628/format/webp)
将图片下载到本地,并且返回保存的路径,然后将路径保存到数据库中,由于数据量较多,可以通过使用多线程的方式缩短处理的时间
线程类
/**
* 使用多线程处理批量图片的保存操作
* @version 1.0.0
*/
public class RunnableSavePicture implements Runnable {
private static final Logger LOGGER = LoggerFactory.getLogger(RunnableSavePicture.class);
// 空List,用于存放已经处理好图片的TrademarkInfoDTO对象
private List<TrademarkInfoDTO> emptySaveDataList;
// 传递过来的包含图片链接的对象
private TrademarkInfoDTO trademarkInfoDTO;
// 保存路径
private String filePath;
public RunnableSavePicture(TrademarkInfoDTO trademarkInfoDTO, List<TrademarkInfoDTO> emptySaveDataList, String filePath) {
this.trademarkInfoDTO = trademarkInfoDTO;
this.emptySaveDataList = emptySaveDataList;
this.filePath = filePath;
}
@Override
public void run() {
try {
String oldUrl = trademarkInfoDTO.getTrademarkImgPath();
if (!StringUtils.isBlank(oldUrl)) {
byte[] imageByte = null;
// 用于请求图片不成功时,再次请求
int count = 0;
while (null == imageByte || imageByte.length < 1024) {
// 通过链接请求图片返回byte[]用于保存
imageByte = HttpUtil.getPicutre(oldUrl);
// 3次失败则放弃
if (count > 3) break;
count++;
}
// 使用UUID生成图片文件名
String fileName = UUID.randomUUID().toString().replaceAll("-", "");
// 将图片保存到本地
saveFile(filePath, imageByte, fileName, "jpg");
// 设置路径
trademarkInfoDTO.setTrademarkImgPath("/icon/" + fileName + ".jpg");
// 添加到空List中(用于保存到数据库)
emptySaveDataList.add(trademarkInfoDTO);
}
} catch (Exception e) {
e.printStackTrace();
LOGGER.debug("==【线程内保存图片操作】==执行异常,注册号:" + trademarkInfoDTO.getTrademarkNum());
LOGGER.debug("==【线程内保存图片操作】==执行异常" + e);
}
}
/*
* @描述:保存图片
* @param:[basePath-存储路径, content-图片byte, fileName-图片名称, fileType-图片类型]
* @return:void
*/
private static void saveFile(String basePath, byte[] content, String fileName, String fileType) throws Exception {
File savePath = new File(basePath);
if (!savePath.exists()) {
savePath.mkdir();
}
File file = new File(savePath.getAbsolutePath() + "/" + fileName + "." + fileType);
FileOutputStream fos = new FileOutputStream(file);
fos.write(content);
fos.close();
}
}
使用方法
public static void main(String[] args) {
// 从数据库中查询出图片链接
List<TrademarkInfoDTO> canSaveDataList = trademarkTemporaryMapper.getCanSaveDataList(adminUid);
// 当前为空List,在线程内处理好图片后,会将传递过去的TrademarkInfoDTO对象放进该List中
List<TrademarkInfoDTO> emptySaveDataList = new ArrayList<>();
long startTime = System.currentTimeMillis();
String filePath = "E:\\icon\\";
int availProcessors = Runtime.getRuntime().availableProcessors();
LOGGER.debug("==【多线程处理图片】==使用线程数:" + (availProcessors * 2));
ExecutorService executorService = Executors.newFixedThreadPool(availProcessors * 2);
for (TrademarkInfoDTO trademarkInfoDTO : canSaveDataList) {
executorService.execute(new RunnableSavePicture(trademarkInfoDTO, emptySaveDataList, filePath));
}
executorService.shutdown();
executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS);
LOGGER.debug("==【多线程处理图片】==图片保存执行完毕,共保存图片:" + emptySaveDataList.size() +
"张,用时:" + (System.currentTimeMillis() - startTime));
// 执行批量插入操作
int count = trademarkTemporaryMapper.saveAllCrawlTrademark(emptySaveDataList);
LOGGER.debug("==【保存数据操作执行完毕】==共保存:" + count);
}
用到的 HttpUtil 工具类
public class HttpUtil {
/*
* @描述:通过图片链接下载图片并返回byte[]
* @param:[url]
* @return:byte[]
*/
public static byte[] getPicutre(String url) throws Exception {
byte[] data;
try {
Request request = new Request.Builder().url(url).build();
com.squareup.okhttp.Response response = CLIENT.newCall(request).execute();
data = response.body().bytes();
} catch (Exception e) {
return null;
}
return data;
}
}
用到okhttp,pom.xml需要导入下面两个jar包
<!-- https://mvnrepository.com/artifact/com.squareup.okhttp/okhttp -->
<dependency>
<groupId>com.squareup.okhttp</groupId>
<artifactId>okhttp</artifactId>
<version>2.7.5</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.squareup.okio/okio -->
<dependency>
<groupId>com.squareup.okio</groupId>
<artifactId>okio</artifactId>
<version>1.13.0</version>
</dependency>
网友评论