yangyoupeng

对ESHTTPClinet做了修改,修复重复创建poolmanager的对象的性能问题,

添加time定时清除http的idle,expire的connect
......@@ -4,17 +4,24 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequest;
import org.apache.http.ParseException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonProcessingException;
......@@ -38,14 +45,38 @@ import com.zhaoonline.common.json.JsonUtils;
* @author zhaoonline)yangyoupeng
*/
public class ESHttpClient {
private static final Logger LOG = LoggerFactory.getLogger(ESHttpClient.class);
private static final String PATH_SEPERATOR = "/";
private String _index;
private String _type;
private CloseableHttpClient httpClient;
private static final AtomicReference<CloseableHttpClient> httpClient = new AtomicReference<CloseableHttpClient>();
private static final Thread monitorThread=createMonitorThread();
private ESClientConfiguration config;
private String path;
private List<HttpHost> hostsList=new ArrayList<HttpHost>();
private static final Timer CONNECTION_MANAGER_TIMER = new Timer(true);
static {
CONNECTION_MANAGER_TIMER.schedule(new TimerTask() {
@Override
public void run() {
try {
final CloseableHttpClient hc = httpClient.get();
if (hc == null) return;
hc.getConnectionManager().closeExpiredConnections();
hc.getConnectionManager().closeIdleConnections(30, TimeUnit.SECONDS);
} catch (Throwable t) {
LOG.error("error closing expired connections", t);
}
}
}, 30000, 5000);
Runtime.getRuntime().addShutdownHook(monitorThread);
}
public ESHttpClient(String index,String type,ESClientConfiguration config){
this._index=index;
this._type=type;
......@@ -53,15 +84,31 @@ public class ESHttpClient {
this.config=config;
}
private static Thread createMonitorThread() {
Thread monitorThread =new Thread(new Runnable() {
@Override
public void run() {
CONNECTION_MANAGER_TIMER.cancel();
LOG.info("pool Manager check timer cancel running");
}
});
return monitorThread;
}
private void concatPathWithIndexType() {
this.path=_index+PATH_SEPERATOR+_type;
}
public void init(){
httpClient =HttpClientFactory.newClient();
getClient();
List<String> hostPosts=config.getHostPorts();
hostsList=buildHostPortListFromString(hostPosts);
}
private CloseableHttpClient getClient() {
if(httpClient.get() ==null){
httpClient.set(HttpClientFactory.newClient());
}
return httpClient.get();
}
......@@ -253,7 +300,7 @@ public class ESHttpClient {
CloseableHttpResponse response=null;
for(HttpHost httpHost:hostsList){
try {
response= httpClient.execute(httpHost, request);
response= getClient().execute(httpHost, request);
if(response !=null){
return response;
}
......@@ -265,7 +312,8 @@ public class ESHttpClient {
}
public void close(){
IOUtils.closeStream(httpClient);
IOUtils.closeStream(getClient());
httpClient.set(null);
}
public boolean deleteDoc(String id) throws JsonParseException, JsonMappingException, UnsupportedOperationException, IOException {
......
package com.zhaoonline.common.es;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.conn.HttpClientConnectionManager;
import org.apache.http.impl.client.CloseableHttpClient;
......@@ -8,8 +9,8 @@ import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
public class HttpClientFactory {
private static Integer SOCKET_TIMEOUT =10000;
private static Integer CONNECTION_TIMEOUT=2000;
private static Integer CONNECTION_TIMEOUT=4000;
public static final CloseableHttpClient newClient() {
final HttpClientBuilder builder = HttpClientBuilder.create();
builder.setConnectionManager(newConnectionManager());
......@@ -21,7 +22,6 @@ public class HttpClientFactory {
CloseableHttpClient httpclient = builder.build();
return httpclient;
}
public static final HttpClientConnectionManager newConnectionManager() {
// 默认支持http和https协议
final PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
......
......@@ -33,17 +33,14 @@ public class TestESClient {
ESHttpClient client= new ESHttpClient(index,type,config);
client.init();
boolean result=false;
for(int i=0;i<100;i++){
Map object=new HashMap<>();
object.put("testKey"+i, "testValue");
String docString=JsonUtils.toJson(object);
boolean result=client.addDoc(""+i,docString);
result=client.addDoc(""+i,docString);
}
// Assert.assertTrue(result);
Assert.assertTrue(result);
}
......@@ -72,7 +69,7 @@ public class TestESClient {
String index="zhaoon1";
String type="test1";
ESClientConfiguration config=new ESClientConfiguration();
config.addHostPorts("127.0.0.1:9200");
config.addHostPorts("192.168.0.162:9200");
ESHttpClient client= new ESHttpClient(index,type,config);
client.init();
String id="1";
......@@ -260,7 +257,7 @@ public class TestESClient {
@Test
public void testBulkAddDate() throws UnsupportedOperationException, IOException{
String index="zhaoon1";
String index="zhaoon2";
String type="test1";
ESClientConfiguration config=new ESClientConfiguration();
config.addHostPorts("192.168.0.162:9200");
......
package com.zhaoonline.common.es;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import org.junit.Assert;
import org.junit.Test;
import com.zhaoonline.common.json.TestDateObject;
public class TestHttpClientFactory {
@Test
public void testConnectEvict() throws InterruptedException{
String index="zhaoon2";
String type="test1";
ESClientConfiguration config=new ESClientConfiguration();
config.addHostPorts("192.168.0.162:9200");
ESHttpClient client= new ESHttpClient(index,type,config);
client.init();
List ids=new ArrayList<>();
List dataList=new ArrayList<>();
for(int i=0;i<100;i++){
TestDateObject o=new TestDateObject();
o.setId(i);
o.setDate(new Date(System.currentTimeMillis()));
dataList.add(o);
ids.add(i);
}
boolean result=false;
try {
result=client.bulkAddDoc(ids,dataList);
Assert.assertTrue(result);
} catch (UnsupportedOperationException | IOException e) {
e.printStackTrace();
}
Thread.sleep(10000);
try {
result=client.bulkAddDoc(ids,dataList);
Assert.assertTrue(result);
} catch (UnsupportedOperationException | IOException e) {
e.printStackTrace();
}
client.close();
try {
result=client.bulkAddDoc(ids,dataList);
Assert.assertTrue(result);
} catch (UnsupportedOperationException | IOException e) {
e.printStackTrace();
}
ESHttpClient client2= new ESHttpClient(index,type,config);
client2.init();
try {
result=client2.bulkAddDoc(ids,dataList);
} catch (UnsupportedOperationException | IOException e) {
e.printStackTrace();
}
Assert.assertTrue(result);
}
@Test
public void testMultiThread(){
int threadCount = 200;
CountDownLatch latch =new CountDownLatch(threadCount);
String index="zhaoon2";
String type="test1";
ESClientConfiguration config=new ESClientConfiguration();
config.addHostPorts("192.168.0.162:9200");
for(int i=0;i<threadCount;i++){
WriteThread wt= new WriteThread(latch,index,type,config,i);
Thread t=new Thread(wt);
t.start();
}
try {
latch.await();
} catch (InterruptedException e) {
e.printStackTrace();
}
System.out.println("it is ok to comme to this step");
}
class WriteThread implements Runnable{
CountDownLatch latch =null;
private String _index;
private String _type;
private ESClientConfiguration config;
private int threadid;
public WriteThread(CountDownLatch latch,String index,String type,ESClientConfiguration config, int i){
this.latch= latch;
this._index=index;
this._type=type;
this.config=config;
this.threadid=i;
}
@Override
public void run() {
System.out.println("thread "+threadid+" begin to running at:"+System.currentTimeMillis());
ESHttpClient client= new ESHttpClient( _index, _type, config);
client.init();
List ids=new ArrayList<>();
List dataList=new ArrayList<>();
for(int i=0;i<100;i++){
TestDateObject o=new TestDateObject();
o.setId(i);
o.setDate(new Date(System.currentTimeMillis()));
dataList.add(o);
ids.add(i);
}
boolean result=false;
try {
result=client.bulkAddDoc(ids,dataList);
System.out.println("thread "+threadid+" end to run result:"+result+" at:"+System.currentTimeMillis());
} catch (UnsupportedOperationException | IOException e) {
e.printStackTrace();
}
this.latch.countDown();
}
}
}