美文网首页
redis 批操作,内存溢出问题解决

redis 批操作,内存溢出问题解决

作者: laod_wh | 来源:发表于2019-09-25 18:07 被阅读0次

    在现场,测试反馈redis报错,原因为:数据过多,导致内存溢出。

    Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
        at java.util.HashMap.newNode(HashMap.java:1742)
        at java.util.HashMap.putVal(HashMap.java:630)
        at java.util.HashMap.put(HashMap.java:611)
        at java.util.HashSet.add(HashSet.java:219)
        at redis.clients.jedis.BuilderFactory$9.build(BuilderFactory.java:140)
        at redis.clients.jedis.BuilderFactory$9.build(BuilderFactory.java:128)
        at redis.clients.jedis.Jedis.keys(Jedis.java:246)
        at com.sumavision.omc.microservice.bo.cache.manager.config.JedisClientComponent.batchClearCache(JedisClientComponent.java:536)
        at com.sumavision.omc.microservice.bo.cache.manager.controller.InitController.initUser(InitController.java:355)
        at com.sumavision.omc.microservice.bo.cache.manager.controller.InitController.initMongoToRedis(InitController.java:140)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.springframework.beans.factory.annotation.InitDestroyAnnotationBeanPostProcessor$LifecycleElement.invoke(InitDestroyAnnotationBeanPostProcessor.java:366)
        at org.springframework.beans.factory.annotation.InitDestroyAnnotationBeanPostProcessor$LifecycleMetadata.invokeInitMethods(InitDestroyAnnotationBeanPostProcessor.java:311)
        at org.springframework.beans.factory.annotation.InitDestroyAnnotationBeanPostProcessor.postProcessBeforeInitialization(InitDestroyAnnotationBeanPostProcessor.java:134)
        at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.applyBeanPostProcessorsBeforeInitialization(AbstractAutowireCapableBeanFactory.java:409)
        at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1626)
        at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:555)
        at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:483)
        at org.springframework.beans.factory.support.AbstractBeanFactory$1.getObject(AbstractBeanFactory.java:312)
        at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:230)
        at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:308)
        at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:197)
        at org.springframework.beans.factory.support.DefaultListableBeanFactory.preInstantiateSingletons(DefaultListableBeanFactory.java:761)
        at org.springframework.context.support.AbstractApplicationContext.finishBeanFactoryInitialization(AbstractApplicationContext.java:867)
        at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:543)
        at org.springframework.boot.context.embedded.EmbeddedWebApplicationContext.refresh(EmbeddedWebApplicationContext.java:122)
        at org.springframework.boot.SpringApplication.refresh(SpringApplication.java:693)
        at org.springframework.boot.SpringApplication.refreshContext(SpringApplication.java:360)
        at org.springframework.boot.SpringApplication.run(SpringApplication.java:303)
    

    经定位,为这块代码报错,由于数据很多导致。

    /**
         * Set<String> set = jedis.keys(key +"*"); 这行代码报错
         * 批量刪除以某字符串为前缀的key
         * @param key
         */
        public  void batchClearCache(String key){
            Jedis jedis = null;
            try {
                
                jedis = jedisClient.getMasterJedis();
                Set<String> set = jedis.keys(key +"*");
                Iterator<String> it = set.iterator();
                while(it.hasNext()){
                    String keyStr = it.next();
                    jedis.del(keyStr);
                }
    
            } catch (Exception e) {
                BaseLogMethod.logError(TAG, "batchClearCache [" + key + "] failed!", JedisClientComponent.class);
                
            } finally {
                if(jedis != null){
                    // 关闭Jedis,对象回到连接池
                    jedis.close();
                }
            }
        }
    

    优化后代码,正常运行!

        public void scan(String key) {
            if(StringUtils.isBlank(key)){
                return;
            }
            
            Jedis jedis = null;
            try {
                ScanParams params = new ScanParams();
                params.match(key+"*");
                params.count(5000);
                String cursor = "0";
                int num = 0;
                jedis = jedisClient.getMasterJedis();
                Pipeline pipelined = jedis.pipelined();
                while (true) {
                    ScanResult scanResult = jedis.scan(cursor,params);
                    List<String> elements = scanResult.getResult();
                    if (elements != null && elements.size() > 0) {
                        for (String s : elements) {
                            pipelined.del(s);
                            
                            num++;
                            if(num > 1000){
                                num = 0;
                                pipelined.sync();// 执行
                            }
                        }
                        if(num > 0){
                            pipelined.sync();// 执行
                        }
                        BaseLogMethod.logDebug(TAG,"scanResult size:{}",elements.size());
                    }
                    cursor = scanResult.getStringCursor();
                    if ("0".equals(cursor)) {
                        break;
                    }
                    pipelined.clear();
                }
                try {
                    pipelined.close();
                } catch (IOException e) {
                    BaseLogMethod.logError(TAG, "cache [" + key + "] failed!", JedisClientComponent.class);
                }
            } catch (Exception e) {
                BaseLogMethod.logError(TAG, "scan [" + key + "] failed!", JedisClientComponent.class);
                
            } finally {
                if(jedis != null){
                    // 关闭Jedis,对象回到连接池
                    jedis.close();
                }
            }
            BaseLogMethod.logDebug(TAG,"batch key success");
        }
    

    新加

    可以加大jvm的运行内存,但是最好还是分批去做处理

    相关文章

      网友评论

          本文标题:redis 批操作,内存溢出问题解决

          本文链接:https://www.haomeiwen.com/subject/nceqyctx.html