美文网首页
redis 批操作,内存溢出问题解决

redis 批操作,内存溢出问题解决

作者: laod_wh | 来源:发表于2019-09-25 18:07 被阅读0次

在现场,测试反馈redis报错,原因为:数据过多,导致内存溢出。

Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
    at java.util.HashMap.newNode(HashMap.java:1742)
    at java.util.HashMap.putVal(HashMap.java:630)
    at java.util.HashMap.put(HashMap.java:611)
    at java.util.HashSet.add(HashSet.java:219)
    at redis.clients.jedis.BuilderFactory$9.build(BuilderFactory.java:140)
    at redis.clients.jedis.BuilderFactory$9.build(BuilderFactory.java:128)
    at redis.clients.jedis.Jedis.keys(Jedis.java:246)
    at com.sumavision.omc.microservice.bo.cache.manager.config.JedisClientComponent.batchClearCache(JedisClientComponent.java:536)
    at com.sumavision.omc.microservice.bo.cache.manager.controller.InitController.initUser(InitController.java:355)
    at com.sumavision.omc.microservice.bo.cache.manager.controller.InitController.initMongoToRedis(InitController.java:140)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at org.springframework.beans.factory.annotation.InitDestroyAnnotationBeanPostProcessor$LifecycleElement.invoke(InitDestroyAnnotationBeanPostProcessor.java:366)
    at org.springframework.beans.factory.annotation.InitDestroyAnnotationBeanPostProcessor$LifecycleMetadata.invokeInitMethods(InitDestroyAnnotationBeanPostProcessor.java:311)
    at org.springframework.beans.factory.annotation.InitDestroyAnnotationBeanPostProcessor.postProcessBeforeInitialization(InitDestroyAnnotationBeanPostProcessor.java:134)
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.applyBeanPostProcessorsBeforeInitialization(AbstractAutowireCapableBeanFactory.java:409)
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1626)
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:555)
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:483)
    at org.springframework.beans.factory.support.AbstractBeanFactory$1.getObject(AbstractBeanFactory.java:312)
    at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:230)
    at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:308)
    at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:197)
    at org.springframework.beans.factory.support.DefaultListableBeanFactory.preInstantiateSingletons(DefaultListableBeanFactory.java:761)
    at org.springframework.context.support.AbstractApplicationContext.finishBeanFactoryInitialization(AbstractApplicationContext.java:867)
    at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:543)
    at org.springframework.boot.context.embedded.EmbeddedWebApplicationContext.refresh(EmbeddedWebApplicationContext.java:122)
    at org.springframework.boot.SpringApplication.refresh(SpringApplication.java:693)
    at org.springframework.boot.SpringApplication.refreshContext(SpringApplication.java:360)
    at org.springframework.boot.SpringApplication.run(SpringApplication.java:303)

经定位,为这块代码报错,由于数据很多导致。

/**
     * Set<String> set = jedis.keys(key +"*"); 这行代码报错
     * 批量刪除以某字符串为前缀的key
     * @param key
     */
    public  void batchClearCache(String key){
        Jedis jedis = null;
        try {
            
            jedis = jedisClient.getMasterJedis();
            Set<String> set = jedis.keys(key +"*");
            Iterator<String> it = set.iterator();
            while(it.hasNext()){
                String keyStr = it.next();
                jedis.del(keyStr);
            }

        } catch (Exception e) {
            BaseLogMethod.logError(TAG, "batchClearCache [" + key + "] failed!", JedisClientComponent.class);
            
        } finally {
            if(jedis != null){
                // 关闭Jedis,对象回到连接池
                jedis.close();
            }
        }
    }

优化后代码,正常运行!

    public void scan(String key) {
        if(StringUtils.isBlank(key)){
            return;
        }
        
        Jedis jedis = null;
        try {
            ScanParams params = new ScanParams();
            params.match(key+"*");
            params.count(5000);
            String cursor = "0";
            int num = 0;
            jedis = jedisClient.getMasterJedis();
            Pipeline pipelined = jedis.pipelined();
            while (true) {
                ScanResult scanResult = jedis.scan(cursor,params);
                List<String> elements = scanResult.getResult();
                if (elements != null && elements.size() > 0) {
                    for (String s : elements) {
                        pipelined.del(s);
                        
                        num++;
                        if(num > 1000){
                            num = 0;
                            pipelined.sync();// 执行
                        }
                    }
                    if(num > 0){
                        pipelined.sync();// 执行
                    }
                    BaseLogMethod.logDebug(TAG,"scanResult size:{}",elements.size());
                }
                cursor = scanResult.getStringCursor();
                if ("0".equals(cursor)) {
                    break;
                }
                pipelined.clear();
            }
            try {
                pipelined.close();
            } catch (IOException e) {
                BaseLogMethod.logError(TAG, "cache [" + key + "] failed!", JedisClientComponent.class);
            }
        } catch (Exception e) {
            BaseLogMethod.logError(TAG, "scan [" + key + "] failed!", JedisClientComponent.class);
            
        } finally {
            if(jedis != null){
                // 关闭Jedis,对象回到连接池
                jedis.close();
            }
        }
        BaseLogMethod.logDebug(TAG,"batch key success");
    }

新加

可以加大jvm的运行内存,但是最好还是分批去做处理

相关文章

网友评论

      本文标题:redis 批操作,内存溢出问题解决

      本文链接:https://www.haomeiwen.com/subject/nceqyctx.html