【JVM】JVM 内存大对象监控和优化实践

-Xms4096M -Xmx4096M -Xmn1024M -XX:MetaspaceSize=256M 
-Djava.security.egd=file:/dev/./urandom -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/data/{runuser}/logs/other

//org.apache.dubbo.remoting.exchange.codec.ExchangeCodec#encodeResponse
protected void encodeResponse(Channel channel, ChannelBuffer buffer, Response res) throws IOException {
    //...省略部分代码
    try {
 
        //1、检查响应大小是否超过 payload，如果超过，则抛出ExceedPayloadLimitException异常
        checkPayload(channel, len);
 
 
    } catch (Throwable t) {
         
        //2、重置buffer
        buffer.writerIndex(savedWriteIndex);
 
        //3、捕获异常后，生成一个新的空响应
        Response r = new Response(res.getId(), res.getVersion());
        r.setStatus(Response.BAD_RESPONSE);
         
        //4、ExceedPayloadLimitException异常，将生成的空响应重新发送一遍
        if (t instanceof ExceedPayloadLimitException) {
            r.setErrorMessage(t.getMessage());
            channel.send(r);
            return;
        }
         
    }
}
 
//org.apache.dubbo.remoting.transport.AbstractCodec#checkPayload
protected static void checkPayload(Channel channel, long size) throws IOException {
    int payload = getPayload(channel);
    boolean overPayload = isOverPayload(payload, size);
    if (overPayload) {
        ExceedPayloadLimitException e = new ExceedPayloadLimitException("Data length too large: " + size + ", max payload: " + payload + ", channel: " + channel);
        logger.error(e);
        throw e;
    }
}

/**
 * 自定义dubbo编码类
 **/
public class MusicDubboCountCodec implements Codec2 {
 
    /**
     * 异常响应池：缓存超过payload大小的responseId
     */
    private static Cache<Long, String> EXCEED_PAYLOAD_LIMIT_CACHE = Caffeine.newBuilder()
        // 缓存总条数
        .maximumSize(100)
        // 过期时间
        .expireAfterWrite(300, TimeUnit.SECONDS)
        // 将value设置为软引用，在OOM前直接淘汰
        .softValues()
        .build();
 
 
    @Override
    public void encode(Channel channel, ChannelBuffer buffer, Object message) throws IOException {
        //1、记录数据编码前的buffer位置
        int writeBefore = null == buffer ? 0 : buffer.writerIndex();
 
        //2、调用原始的编码方法
        dubboCountCodec.encode(channel, buffer, message);
 
        //3、检查&记录超过payload的信息
        checkOverPayload(message);
 
        //4、计算对象长度
        int writeAfter = null == buffer ? 0 : buffer.writerIndex();    
        int length = writeAfter - writeBefore;
 
        //5、超过告警阈值，进行日志打印处理
        warningLengthTooLong(length, message);
    }
 
    //校验response是否超过payload，超过了，缓存id
    private void checkOverPayload(Object message){
        if(!(message instanceof Response)){
            return;
        }
        Response response = (Response) message;
 
        //3.1、新的发送过程：通过状态码BAD_RESPONSE与错误信息识别出空响应，并记录响应id
        if(Response.BAD_RESPONSE == response.getStatus() && StrUtil.contains(response.getErrorMessage(), OVER_PAYLOAD_ERROR_MESSAGE)){          
            EXCEED_PAYLOAD_LIMIT_CACHE.put(response.getId(), response.getErrorMessage());
            return;
        }
 
        //3.2、原先的发送过程：通过异常池识别出超过payload的响应，打印有用的信息
        if(Response.OK == response.getStatus() &&  EXCEED_PAYLOAD_LIMIT_CACHE.getIfPresent(response.getId()) != null){      
            String responseMessage = getResponseMessage(response);
            log.warn("dubbo序列化对象大小超过payload,errorMsg is {},response is {}", EXCEED_PAYLOAD_LIMIT_CACHE.getIfPresent(response.getId()),responseMessage);
        }
    }
     
}