下面是我的代码和线程转储。我不知道为什么 CPU 会达到 100%。有人可以帮忙吗？下面的方法是可疑的，我只是想解析结果集并使用 Apache CSV (commons-csv-1.5.jar) 将其写入 csv。当我评论调用此方法的行时，cpu 保持在 3%。
public static void writeResultSetToFile(ResultSet resultSet, String fileName) {
BufferedWriter writer = null;
CSVPrinter csvPrinter = null;
//If the file with the same filename already exist, a date stamp is appended to the end of the file.
if(checkIfFileExist(fileName)) {
LOGGER.info("FILE EXIST:"+fileName);
String fileNamePostFix = new SimpleDateFormat(Constants.FORMAT_yyyyMMddHHmm).format(new Date());
fileName=fileName.concat(Constants.UNDERSCORE).concat(fileNamePostFix);
LOGGER.info("WRITING TO FILE: "+fileName);
}
try {
ResultSetMetaData metadata = resultSet.getMetaData();
writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName),StandardCharsets.UTF_8));
//Fetch the column header from the metadata and inserts to an arrayList
columnCount = metadata.getColumnCount();
List<String> headerList = new ArrayList<String>(0);
for (int i = 1; i <= columnCount; i++) {
headerList.add(metadata.getColumnName(i));
}
String[] headerArray = new String[headerList.size()];
headerArray = headerList.toArray(headerArray);
//Creates a csv printer with the column names fetched from the database
csvPrinter = new CSVPrinter(writer, CSVFormat.DEFAULT.withHeader(headerArray).withDelimiter(Constants.C_DELIMITER));
recordCount = 0;
List<String> valueList = new ArrayList<String>(0);
while (resultSet.next()) {
recordCount++;
for (int i = 1; i <= columnCount; i++) {
valueList.add(resultSet.getString(i));
}
csvPrinter.printRecord(valueList);
valueList = new ArrayList<String>(0);
}
} catch (SQLException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
finally {
if (writer != null) {
try {
writer.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (csvPrinter != null) {
try {
csvPrinter.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
线程转储
Full thread dump OpenJDK 64-Bit Server VM (24.171-b01 mixed mode):
"pool-2-thread-1" prio=10 tid=0x00007f2eb839d000 nid=0x264e runnable [0x00007f2ea6dc4000]
java.lang.Thread.State: RUNNABLE
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.read(SocketInputStream.java:153)
at java.net.SocketInputStream.read(SocketInputStream.java:122)
at java.io.BufferedInputStream.fill(BufferedInputStream.java:235)
at java.io.BufferedInputStream.read1(BufferedInputStream.java:275)
at java.io.BufferedInputStream.read(BufferedInputStream.java:334)
- locked <0x00000000e01bc220> (a java.io.BufferedInputStream)
at org.apache.thrift.transport.TIOStreamTransport.read(TIOStreamTransport.java:127)
at org.apache.thrift.transport.TTransport.readAll(TTransport.java:84)
at org.apache.thrift.transport.TSaslTransport.readLength(TSaslTransport.java:346)
at org.apache.thrift.transport.TSaslTransport.readFrame(TSaslTransport.java:423)
at org.apache.thrift.transport.TSaslTransport.read(TSaslTransport.java:405)
at org.apache.thrift.transport.TSaslClientTransport.read(TSaslClientTransport.java:37)
at org.apache.thrift.transport.TTransport.readAll(TTransport.java:84)
at org.apache.thrift.protocol.TBinaryProtocol.readAll(TBinaryProtocol.java:378)
at org.apache.thrift.protocol.TBinaryProtocol.readI32(TBinaryProtocol.java:297)
at org.apache.thrift.protocol.TBinaryProtocol.readMessageBegin(TBinaryProtocol.java:204)
at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:69)
at org.apache.hive.service.cli.thrift.TCLIService$Client.recv_FetchResults(TCLIService.java:515)
at org.apache.hive.service.cli.thrift.TCLIService$Client.FetchResults(TCLIService.java:502)
at com.cloudera.hive.hivecommon.api.HS2Client.fetchNRows(HS2Client.java:321)
at com.cloudera.hive.hive.api.ExtendedHS2Client.fetchNRows(ExtendedHS2Client.java:499)
at com.cloudera.hive.hivecommon.api.HS2Client.fetchRows(HS2Client.java:301)
at com.cloudera.hive.hivecommon.dataengine.BackgroundFetcher.run(BackgroundFetcher.java:138)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:473)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:622)
at java.lang.Thread.run(Thread.java:748)
"Service Thread" daemon prio=10 tid=0x00007f2eb80b4800 nid=0x2562 runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread1" daemon prio=10 tid=0x00007f2eb80b2000 nid=0x2561 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread0" daemon prio=10 tid=0x00007f2eb80af000 nid=0x2560 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Signal Dispatcher" daemon prio=10 tid=0x00007f2eb80ad000 nid=0x255f waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Finalizer" daemon prio=10 tid=0x00007f2eb807f800 nid=0x255e in Object.wait() [0x00007f2eb45f4000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000000e00247f0> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135)
- locked <0x00000000e00247f0> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151)
at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209)
"Reference Handler" daemon prio=10 tid=0x00007f2eb807d800 nid=0x255d in Object.wait() [0x00007f2eb46f5000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000000e000e1c8> (a java.lang.ref.Reference$Lock)
at java.lang.Object.wait(Object.java:503)
at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133)
- locked <0x00000000e000e1c8> (a java.lang.ref.Reference$Lock)
"main" prio=10 tid=0x00007f2eb800b800 nid=0x2557 runnable [0x00007f2ec05fb000]
java.lang.Thread.State: RUNNABLE
at java.lang.String.split(String.java:2289)
at java.lang.String.split(String.java:2355)
at com.cloudera.hive.hivecommon.dataengine.HiveJDBCQueryAnalyserUtils.queryAnalysis(HiveJDBCQueryAnalyserUtils.java:49)
at com.cloudera.hive.hivecommon.api.HS2Buffer.getData(HS2Buffer.java:181)
at com.cloudera.hive.hivecommon.api.HS2Client.getData(HS2Client.java:705)
at com.cloudera.hive.hivecommon.dataengine.HiveJDBCResultSet.getData(HiveJDBCResultSet.java:265)
at com.cloudera.hive.jdbc.common.SForwardResultSet.getData(SForwardResultSet.java:4590)
at com.cloudera.hive.jdbc.common.SForwardResultSet.getString(SForwardResultSet.java:2138)
at xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.utils.FileUtils.writeResultSetToFile(FileUtils.java:153)
at xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.utils.DatabaseUtils.executeQueryAndWriteToFile(DatabaseUtils.java:135)
at xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.processor.JDBCProcessor.processCustomQueries(JDBCProcessor.java:84)
at xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.processor.JDBCProcessor.process(JDBCProcessor.java:47)
at xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.main.App.main(App.java:49)
"VM Thread" prio=10 tid=0x00007f2eb8077000 nid=0x255c runnable
"GC task thread#0 (ParallelGC)" prio=10 tid=0x00007f2eb8021000 nid=0x2558 runnable
"GC task thread#1 (ParallelGC)" prio=10 tid=0x00007f2eb8023000 nid=0x2559 runnable
"GC task thread#2 (ParallelGC)" prio=10 tid=0x00007f2eb8025000 nid=0x255a runnable
"GC task thread#3 (ParallelGC)" prio=10 tid=0x00007f2eb8027000 nid=0x255b runnable
"VM Periodic Task Thread" prio=10 tid=0x00007f2eb80bf800 nid=0x2563 waiting on condition
JNI global references: 234
Heap
PSYoungGen total 160768K, used 126191K [0x00000000f5500000, 0x00000000ffd80000, 0x0000000100000000)
eden space 149504K, 84% used [0x00000000f5500000,0x00000000fd033ef8,0x00000000fe700000)
from space 11264K, 0% used [0x00000000fe700000,0x00000000fe708000,0x00000000ff200000)
to space 10240K, 0% used [0x00000000ff380000,0x00000000ff380000,0x00000000ffd80000)
ParOldGen total 349184K, used 192583K [0x00000000e0000000, 0x00000000f5500000, 0x00000000f5500000)
object space 349184K, 55% used [0x00000000e0000000,0x00000000ebc11df8,0x00000000f5500000)
PSPermGen total 21504K, used 12023K [0x00000000d5a00000, 0x00000000d6f00000, 0x00000000e0000000)
object space 21504K, 55% used [0x00000000d5a00000,0x00000000d65bde38,0x00000000d6f00000)
最佳答案
好吧，这不是一个答案，而是一些需要空间的提醒:
1) 您正在使用 ArrayList，从零容量开始...它必须经常重新调整其缓冲区的大小，因为 arraylist 大小的增长如下: 0 -> 1 -> 2 -> 4 -> 7 -> 11 -> 17 。
所以而不是 List<String> valueList = new ArrayList<String>(0);使用List<String> headerList = new ArrayList<String>(columnCount);
而不是valueList = new ArrayList<String>(0);使用valueList.clear();
2)你应该使用resource-try-catch，这样更容易处理。
3) 代码 - 或者更确切地说:幕后 - 是大量的解析...解析 SQL 结果，然后“解析”数据(必须以 CSV 兼容的方式转义)。不要低估这一点!更重要的是，如果此 CSVPrinter 执行一些附加格式设置，例如附加间距以保持文本文件看起来像表格等。
4)此外，Apache 库并不因其速度而闻名，也不因其资源效率而闻名!
5)手动刷新(如评论中所述)对性能不利!
6) 从表面上看，你的方法writeResultSetToFile()从循环中调用。我不知道，但如果你告诉我们“3% 使用率”，这听起来像是一个持续的任务，即一个循环。因此，假设存在一个循环，并且该循环直接导致 3% 的 CPU 使用率，那么它似乎每秒循环数千次。
如果您现在在每次调用时检索并存储两百万行文本，这将减慢整个程序的速度。相当。
也许您不应该经常写入该文件？也许一分钟一次就足够了？但通常，您可能会考虑使用与主循环并行运行的解耦线程。
关于Java进程CPU占用率高，我们在Stack Overflow上找到一个类似的问题： https://stackoverflow.com/questions/50223958/