問題描述

使用StreamLoad寫入Starrocks報錯，報這個錯誤:Caused by: org.apache.http.ProtocolException: Content-Length header already present

代碼案例

引入依賴

        <!--  Starrocks使用StreamLoad發(fā)送Http請求   -->
        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpclient</artifactId>
            <version>4.5.3</version>
        </dependency>

具體demo

package com.song.starrocks;

import org.apache.commons.codec.binary.Base64;
import org.apache.http.HttpHeaders;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultRedirectStrategy;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.UUID;

public class StarRocksStreamLoad {
    private final static String STARROCKS_HOST = "localhost";
    private final static String STARROCKS_DB = "starrocks_demo";
    private final static String STARROCKS_TABLE = "bitmap02";
    private final static String STARROCKS_USER = "root";
    private final static String STARROCKS_PASSWORD = "";
    private final static int STARROCKS_HTTP_PORT = 8038;

    public void sendData(String content) throws Exception {
        //String yyyyy = "id,name,age,age=to_bitmap(age)";
        String yyyyy = "id,process_time,bkey,bvalue,bvalue=to_bitmap(bvalue),create_time";
        final String loadUrl = String.format("http://%s:%s/api/%s/%s/_stream_load",
                STARROCKS_HOST,
                STARROCKS_HTTP_PORT,
                STARROCKS_DB,
                STARROCKS_TABLE);

        final HttpClientBuilder httpClientBuilder = null;

        int TIMEOUT = 3000;
        try (CloseableHttpClient client = httpClientBuilder.build()) {
            HttpPut put = new HttpPut(loadUrl);
            RequestConfig build = RequestConfig.custom()
                    .setAuthenticationEnabled(true)
                    .setCircularRedirectsAllowed(true)
                    .setRedirectsEnabled(true)
                    .setRelativeRedirectsAllowed(true)
                    .setExpectContinueEnabled(true)
                    .setConnectTimeout(30000).setConnectionRequestTimeout(TIMEOUT)
                    .setSocketTimeout(TIMEOUT).build();
            put.setConfig(build);
            StringEntity entity = new StringEntity(content, "UTF-8");
            put.setHeader(HttpHeaders.EXPECT, "100-continue");
            put.setHeader(HttpHeaders.AUTHORIZATION, basicAuthHeader(STARROCKS_USER, STARROCKS_PASSWORD));
            put.setHeader("columns", yyyyy);
            put.setHeader("column_separator", "\t");
            put.setHeader("max_filter_ratio", "0.2");
            put.setHeader("label", UUID.randomUUID().toString());
            put.setEntity(entity);

            try (CloseableHttpResponse response = client.execute(put)) {
                String loadResult = "";
                if (response.getEntity() != null) {
                    loadResult = EntityUtils.toString(response.getEntity());
                }
                final int statusCode = response.getStatusLine().getStatusCode();
                // statusCode 200 just indicates that starrocks be service is ok, not stream load
                // you should see the output content to find whether stream load is success
                if (statusCode != 200) {
                    throw new IOException(
                            String.format("Stream load failed, statusCode=%s load result=%s", statusCode, loadResult));
                }
                System.out.println("data {}" + loadResult);
            }
        }
    }

    private String basicAuthHeader(String username, String password) {
        final String tobeEncode = username + ":" + password;
        byte[] encoded = Base64.encodeBase64(tobeEncode.getBytes(StandardCharsets.UTF_8));
        return "Basic " + new String(encoded);
    }

}

測試類

注意:因為ExtRoaringBitmap是自己封裝的，所以需要換成bitmap或者RortingBitmap

    @Test
    public void testBitMapWriteStarrocks() throws Exception {
        ExtRoaringBitmap extRoaringBitmap = new ExtRoaringBitmap();
        extRoaringBitmap.set(111111);

        LongIterator longIterator = extRoaringBitmap.longIterator();
        StringBuilder stringBuilder = new StringBuilder();

        while (longIterator.hasNext()) {

            // 生成id
            Long id = 100;

            // 獲取ExtRoaringBitmap中的具體數據，也就是獲取long類型的offset
            long bitmapValue = longIterator.next();

            Date date = new Date();
            String currentDate = sdf.format(date);
            String bitmapKey = "張三" + id;

            String oneRow = id + "\t" + currentDate + "\t" + bitmapKey + "\t" + bitmapValue + "\t" + currentDate + "\n";
            stringBuilder.append(oneRow);
        }
        stringBuilder.deleteCharAt(stringBuilder.length() - 1);
        String loadData = stringBuilder.toString();
        System.out.println("loadData :{} " + loadData);

        StarRocksStreamLoad starrocksStreamLoad = new StarRocksStreamLoad();
        starrocksStreamLoad.sendData(loadData);

    }

報錯截圖

文章來源地址http://www.zghlxwxcb.cn/news/detail-679773.html

分析錯誤原因

     因為代碼執(zhí)行是在一個父項目中調用的子項目，把寫入Starrocks這塊封裝為一個工具類，通過其他的項目來調用， 如果個給Starrocks單獨一個項目的話，是可以正常執(zhí)行的 ， 一旦放入到整個項目中，通過其他的項目來進行調用，就會報錯，經過排查 ，錯誤的原因是因為父項目中也有用到依賴httpclient這個依賴，但是版本不是4.5.3的，是其他的版本 ，雖然我子項目中有這個httpclient4.5.3的依賴，但是在項目啟動的時候，父項目的依賴把子項目的依賴覆蓋了，導致無法使用httpclient4.5.3的依賴，所以項目一直報Caused by: org.apache.http.ProtocolException: Content-Length header already present。

解決辦法

1.在idea中安裝maven的插件:Maven Helper ,通過這個可以查到里面所有的依賴;
2.在父項目中也添加這個httpclient4.5.3的依賴就可以正常寫入數據了;

到了這里，關于使用StreamLold寫入 Starrocks報錯：Caused by org的文章就介紹完了。如果您還想了解更多內容，請在右上角搜索TOY模板網以前的文章或繼續(xù)瀏覽下面的相關文章，希望大家以后多多支持TOY模板網！