Toggle navigation
Toggle navigation
This project
Loading...
Sign in
yangyoupeng
/
zhaoonline-utils
Go to a project
Toggle navigation
Toggle navigation pinning
Projects
Groups
Snippets
Help
Project
Activity
Repository
Pipelines
Graphs
Issues
0
Merge Requests
0
Wiki
Network
Create a new issue
Builds
Commits
Authored by
yangyoupeng
2016-11-03 16:16:34 +0800
Browse Files
Options
Browse Files
Download
Email Patches
Plain Diff
Commit
f7e120224c4d559158423a412ba849f4e7b0e670
f7e12022
1 parent
63868c77
对ESHTTPClinet做了修改,修复重复创建poolmanager的对象的性能问题,
添加time定时清除http的idle,expire的connect
Show whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
188 additions
and
16 deletions
src/main/java/com/zhaoonline/common/es/ESHttpClient.java
src/main/java/com/zhaoonline/common/es/HttpClientFactory.java
src/test/java/com/zhaoonline/common/es/TestESClient.java
src/test/java/com/zhaoonline/common/es/TestHttpClientFactory.java
src/main/java/com/zhaoonline/common/es/ESHttpClient.java
View file @
f7e1202
...
...
@@ -4,17 +4,24 @@ import java.io.IOException;
import
java.util.ArrayList
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.Timer
;
import
java.util.TimerTask
;
import
java.util.concurrent.TimeUnit
;
import
java.util.concurrent.atomic.AtomicReference
;
import
org.apache.http.HttpEntity
;
import
org.apache.http.HttpHost
;
import
org.apache.http.HttpRequest
;
import
org.apache.http.ParseException
;
import
org.apache.http.client.HttpClient
;
import
org.apache.http.client.methods.CloseableHttpResponse
;
import
org.apache.http.client.methods.RequestBuilder
;
import
org.apache.http.entity.ContentType
;
import
org.apache.http.entity.StringEntity
;
import
org.apache.http.impl.client.CloseableHttpClient
;
import
org.apache.http.util.EntityUtils
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
com.fasterxml.jackson.core.JsonParseException
;
import
com.fasterxml.jackson.core.JsonProcessingException
;
...
...
@@ -38,14 +45,38 @@ import com.zhaoonline.common.json.JsonUtils;
* @author zhaoonline)yangyoupeng
*/
public
class
ESHttpClient
{
private
static
final
Logger
LOG
=
LoggerFactory
.
getLogger
(
ESHttpClient
.
class
);
private
static
final
String
PATH_SEPERATOR
=
"/"
;
private
String
_index
;
private
String
_type
;
private
CloseableHttpClient
httpClient
;
private
static
final
AtomicReference
<
CloseableHttpClient
>
httpClient
=
new
AtomicReference
<
CloseableHttpClient
>();
private
static
final
Thread
monitorThread
=
createMonitorThread
();
private
ESClientConfiguration
config
;
private
String
path
;
private
List
<
HttpHost
>
hostsList
=
new
ArrayList
<
HttpHost
>();
private
static
final
Timer
CONNECTION_MANAGER_TIMER
=
new
Timer
(
true
);
static
{
CONNECTION_MANAGER_TIMER
.
schedule
(
new
TimerTask
()
{
@Override
public
void
run
()
{
try
{
final
CloseableHttpClient
hc
=
httpClient
.
get
();
if
(
hc
==
null
)
return
;
hc
.
getConnectionManager
().
closeExpiredConnections
();
hc
.
getConnectionManager
().
closeIdleConnections
(
30
,
TimeUnit
.
SECONDS
);
}
catch
(
Throwable
t
)
{
LOG
.
error
(
"error closing expired connections"
,
t
);
}
}
},
30000
,
5000
);
Runtime
.
getRuntime
().
addShutdownHook
(
monitorThread
);
}
public
ESHttpClient
(
String
index
,
String
type
,
ESClientConfiguration
config
){
this
.
_index
=
index
;
this
.
_type
=
type
;
...
...
@@ -53,15 +84,31 @@ public class ESHttpClient {
this
.
config
=
config
;
}
private
static
Thread
createMonitorThread
()
{
Thread
monitorThread
=
new
Thread
(
new
Runnable
()
{
@Override
public
void
run
()
{
CONNECTION_MANAGER_TIMER
.
cancel
();
LOG
.
info
(
"pool Manager check timer cancel running"
);
}
});
return
monitorThread
;
}
private
void
concatPathWithIndexType
()
{
this
.
path
=
_index
+
PATH_SEPERATOR
+
_type
;
}
public
void
init
(){
httpClient
=
HttpClientFactory
.
new
Client
();
get
Client
();
List
<
String
>
hostPosts
=
config
.
getHostPorts
();
hostsList
=
buildHostPortListFromString
(
hostPosts
);
}
private
CloseableHttpClient
getClient
()
{
if
(
httpClient
.
get
()
==
null
){
httpClient
.
set
(
HttpClientFactory
.
newClient
());
}
return
httpClient
.
get
();
}
...
...
@@ -253,7 +300,7 @@ public class ESHttpClient {
CloseableHttpResponse
response
=
null
;
for
(
HttpHost
httpHost:
hostsList
){
try
{
response
=
httpClient
.
execute
(
httpHost
,
request
);
response
=
getClient
()
.
execute
(
httpHost
,
request
);
if
(
response
!=
null
){
return
response
;
}
...
...
@@ -265,7 +312,8 @@ public class ESHttpClient {
}
public
void
close
(){
IOUtils
.
closeStream
(
httpClient
);
IOUtils
.
closeStream
(
getClient
());
httpClient
.
set
(
null
);
}
public
boolean
deleteDoc
(
String
id
)
throws
JsonParseException
,
JsonMappingException
,
UnsupportedOperationException
,
IOException
{
...
...
src/main/java/com/zhaoonline/common/es/HttpClientFactory.java
View file @
f7e1202
package
com
.
zhaoonline
.
common
.
es
;
import
org.apache.http.client.config.RequestConfig
;
import
org.apache.http.conn.HttpClientConnectionManager
;
import
org.apache.http.impl.client.CloseableHttpClient
;
...
...
@@ -8,7 +9,7 @@ import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
public
class
HttpClientFactory
{
private
static
Integer
SOCKET_TIMEOUT
=
10000
;
private
static
Integer
CONNECTION_TIMEOUT
=
2
000
;
private
static
Integer
CONNECTION_TIMEOUT
=
4
000
;
public
static
final
CloseableHttpClient
newClient
()
{
final
HttpClientBuilder
builder
=
HttpClientBuilder
.
create
();
...
...
@@ -21,7 +22,6 @@ public class HttpClientFactory {
CloseableHttpClient
httpclient
=
builder
.
build
();
return
httpclient
;
}
public
static
final
HttpClientConnectionManager
newConnectionManager
()
{
// 默认支持http和https协议
final
PoolingHttpClientConnectionManager
cm
=
new
PoolingHttpClientConnectionManager
();
...
...
src/test/java/com/zhaoonline/common/es/TestESClient.java
View file @
f7e1202
...
...
@@ -33,17 +33,14 @@ public class TestESClient {
ESHttpClient
client
=
new
ESHttpClient
(
index
,
type
,
config
);
client
.
init
();
boolean
result
=
false
;
for
(
int
i
=
0
;
i
<
100
;
i
++){
Map
object
=
new
HashMap
<>();
object
.
put
(
"testKey"
+
i
,
"testValue"
);
String
docString
=
JsonUtils
.
toJson
(
object
);
boolean
result
=
client
.
addDoc
(
""
+
i
,
docString
);
result
=
client
.
addDoc
(
""
+
i
,
docString
);
}
// Assert.assertTrue(result);
Assert
.
assertTrue
(
result
);
}
...
...
@@ -72,7 +69,7 @@ public class TestESClient {
String
index
=
"zhaoon1"
;
String
type
=
"test1"
;
ESClientConfiguration
config
=
new
ESClientConfiguration
();
config
.
addHostPorts
(
"1
27.0.0.1
:9200"
);
config
.
addHostPorts
(
"1
92.168.0.162
:9200"
);
ESHttpClient
client
=
new
ESHttpClient
(
index
,
type
,
config
);
client
.
init
();
String
id
=
"1"
;
...
...
@@ -260,7 +257,7 @@ public class TestESClient {
@Test
public
void
testBulkAddDate
()
throws
UnsupportedOperationException
,
IOException
{
String
index
=
"zhaoon
1
"
;
String
index
=
"zhaoon
2
"
;
String
type
=
"test1"
;
ESClientConfiguration
config
=
new
ESClientConfiguration
();
config
.
addHostPorts
(
"192.168.0.162:9200"
);
...
...
src/test/java/com/zhaoonline/common/es/TestHttpClientFactory.java
0 → 100644
View file @
f7e1202
package
com
.
zhaoonline
.
common
.
es
;
import
java.io.IOException
;
import
java.util.ArrayList
;
import
java.util.Date
;
import
java.util.List
;
import
java.util.concurrent.CountDownLatch
;
import
org.junit.Assert
;
import
org.junit.Test
;
import
com.zhaoonline.common.json.TestDateObject
;
public
class
TestHttpClientFactory
{
@Test
public
void
testConnectEvict
()
throws
InterruptedException
{
String
index
=
"zhaoon2"
;
String
type
=
"test1"
;
ESClientConfiguration
config
=
new
ESClientConfiguration
();
config
.
addHostPorts
(
"192.168.0.162:9200"
);
ESHttpClient
client
=
new
ESHttpClient
(
index
,
type
,
config
);
client
.
init
();
List
ids
=
new
ArrayList
<>();
List
dataList
=
new
ArrayList
<>();
for
(
int
i
=
0
;
i
<
100
;
i
++){
TestDateObject
o
=
new
TestDateObject
();
o
.
setId
(
i
);
o
.
setDate
(
new
Date
(
System
.
currentTimeMillis
()));
dataList
.
add
(
o
);
ids
.
add
(
i
);
}
boolean
result
=
false
;
try
{
result
=
client
.
bulkAddDoc
(
ids
,
dataList
);
Assert
.
assertTrue
(
result
);
}
catch
(
UnsupportedOperationException
|
IOException
e
)
{
e
.
printStackTrace
();
}
Thread
.
sleep
(
10000
);
try
{
result
=
client
.
bulkAddDoc
(
ids
,
dataList
);
Assert
.
assertTrue
(
result
);
}
catch
(
UnsupportedOperationException
|
IOException
e
)
{
e
.
printStackTrace
();
}
client
.
close
();
try
{
result
=
client
.
bulkAddDoc
(
ids
,
dataList
);
Assert
.
assertTrue
(
result
);
}
catch
(
UnsupportedOperationException
|
IOException
e
)
{
e
.
printStackTrace
();
}
ESHttpClient
client2
=
new
ESHttpClient
(
index
,
type
,
config
);
client2
.
init
();
try
{
result
=
client2
.
bulkAddDoc
(
ids
,
dataList
);
}
catch
(
UnsupportedOperationException
|
IOException
e
)
{
e
.
printStackTrace
();
}
Assert
.
assertTrue
(
result
);
}
@Test
public
void
testMultiThread
(){
int
threadCount
=
200
;
CountDownLatch
latch
=
new
CountDownLatch
(
threadCount
);
String
index
=
"zhaoon2"
;
String
type
=
"test1"
;
ESClientConfiguration
config
=
new
ESClientConfiguration
();
config
.
addHostPorts
(
"192.168.0.162:9200"
);
for
(
int
i
=
0
;
i
<
threadCount
;
i
++){
WriteThread
wt
=
new
WriteThread
(
latch
,
index
,
type
,
config
,
i
);
Thread
t
=
new
Thread
(
wt
);
t
.
start
();
}
try
{
latch
.
await
();
}
catch
(
InterruptedException
e
)
{
e
.
printStackTrace
();
}
System
.
out
.
println
(
"it is ok to comme to this step"
);
}
class
WriteThread
implements
Runnable
{
CountDownLatch
latch
=
null
;
private
String
_index
;
private
String
_type
;
private
ESClientConfiguration
config
;
private
int
threadid
;
public
WriteThread
(
CountDownLatch
latch
,
String
index
,
String
type
,
ESClientConfiguration
config
,
int
i
){
this
.
latch
=
latch
;
this
.
_index
=
index
;
this
.
_type
=
type
;
this
.
config
=
config
;
this
.
threadid
=
i
;
}
@Override
public
void
run
()
{
System
.
out
.
println
(
"thread "
+
threadid
+
" begin to running at:"
+
System
.
currentTimeMillis
());
ESHttpClient
client
=
new
ESHttpClient
(
_index
,
_type
,
config
);
client
.
init
();
List
ids
=
new
ArrayList
<>();
List
dataList
=
new
ArrayList
<>();
for
(
int
i
=
0
;
i
<
100
;
i
++){
TestDateObject
o
=
new
TestDateObject
();
o
.
setId
(
i
);
o
.
setDate
(
new
Date
(
System
.
currentTimeMillis
()));
dataList
.
add
(
o
);
ids
.
add
(
i
);
}
boolean
result
=
false
;
try
{
result
=
client
.
bulkAddDoc
(
ids
,
dataList
);
System
.
out
.
println
(
"thread "
+
threadid
+
" end to run result:"
+
result
+
" at:"
+
System
.
currentTimeMillis
());
}
catch
(
UnsupportedOperationException
|
IOException
e
)
{
e
.
printStackTrace
();
}
this
.
latch
.
countDown
();
}
}
}
Please
register
or
login
to post a comment