磁盘空间不足导致HiveSession创建失败
问题描述
Hive的服务器（Linux）的磁盘空间不足，会导致与hive建立session失败。最终导致使用到hive服务的Spark任务和Hive任务失败。
堆栈关键字
org.apache.hive.service.cli.HiveSQLException: Failed to open new session: java.lang.RuntimeException: java.lang.RuntimeException: java.lang.RuntimeException: java.io.IOException: No space left on device。
具体错误报文：
[2024-06-28 00:04:28,193][ERROR][From MonitorThreadFactory's Monitor-Worker-3][com.suning.base.monitor.Spark2Monitor:47] Spark2Monitor Exception.
org.apache.hive.service.cli.HiveSQLException: Failed to open new session: java.lang.RuntimeException: java.lang.RuntimeException: java.lang.RuntimeException: java.io.IOException: No space left on device
at org.apache.hive.jdbc.Utils.verifySuccess(Utils.java:256) ~[hive-jdbc-1.2.1.jar:1.2.1]
at org.apache.hive.jdbc.Utils.verifySuccess(Utils.java:247) ~[hive-jdbc-1.2.1.jar:1.2.1]
at org.apache.hive.jdbc.HiveConnection.openSession(HiveConnection.java:586) ~[hive-jdbc-1.2.1.jar:1.2.1]
at org.apache.hive.jdbc.HiveConnection.<init>(HiveConnection.java:192) ~[hive-jdbc-1.2.1.jar:1.2.1]
at org.apache.hive.jdbc.HiveDriver.connect(HiveDriver.java:105) ~[hive-jdbc-1.2.1.jar:1.2.1]
at java.sql.DriverManager.getConnection(DriverManager.java:664) ~[?:1.8.0_131]
at java.sql.DriverManager.getConnection(DriverManager.java:247) ~[?:1.8.0_131]
at com.suning.base.components.hive.HiveClientHelper.getConnection(HiveClientHelper.java:40) ~[base-monitoring-service-1.1.8.jar:1.1.8]
at com.suning.base.components.hive.HiveClientHelper.getConnection(HiveClientHelper.java:36) ~[base-monitoring-service-1.1.8.jar:1.1.8]
at com.suning.base.components.hive.HiveClientHelper.getConnection(HiveClientHelper.java:32) ~[base-monitoring-service-1.1.8.jar:1.1.8]
at com.suning.base.monitor.Spark2Monitor.monitor(Spark2Monitor.java:38) ~[base-monitoring-service-1.1.8.jar:1.1.8]
at com.suning.base.BaseMonitoringService$1.lambda$run$0(BaseMonitoringService.java:264) ~[base-monitoring-service-1.1.8.jar:1.1.8]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_131]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [?:1.8.0_131]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [?:1.8.0_131]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_131]
Caused by: org.apache.hive.service.cli.HiveSQLException: Failed to open new session: java.lang.RuntimeException: java.lang.RuntimeException: java.lang.RuntimeException: java.io.IOException: No space left on device
at org.apache.hive.service.cli.session.SessionManager.openSession(SessionManager.java:266) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.CLIService.openSessionWithImpersonation(CLIService.java:204) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.thrift.ThriftCLIService.getSessionHandle(ThriftCLIService.java:404) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.thrift.ThriftCLIService.OpenSession(ThriftCLIService.java:299) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.thrift.TCLIService$Processor$OpenSession.getResult(TCLIService.java:1253) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.thrift.TCLIService$Processor$OpenSession.getResult(TCLIService.java:1238) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39) ~[libthrift-0.9.2.jar:0.9.2]
at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39) ~[libthrift-0.9.2.jar:0.9.2]
at org.apache.hive.service.auth.TSetIpAddressProcessor.process(TSetIpAddressProcessor.java:56) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:285) ~[libthrift-0.9.2.jar:0.9.2]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_131]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_131]
... 1 more
Caused by: java.lang.RuntimeException: java.lang.RuntimeException: java.lang.RuntimeException: java.io.IOException: No space left on device
at org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:83) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.session.HiveSessionProxy.access$000(HiveSessionProxy.java:36) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.session.HiveSessionProxy$1.run(HiveSessionProxy.java:63) ~[hive-service-1.2.1.jar:1.2.1]
at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_131]
at javax.security.auth.Subject.doAs(Subject.java:422) ~[?:1.8.0_131]
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1836) ~[hadoop-common-2.7.3.jar:?]
at org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:59) ~[hive-service-1.2.1.jar:1.2.1]
at com.sun.proxy.$Proxy20.open(Unknown Source) ~[?:?]
at org.apache.hive.service.cli.session.SessionManager.openSession(SessionManager.java:258) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.CLIService.openSessionWithImpersonation(CLIService.java:204) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.thrift.ThriftCLIService.getSessionHandle(ThriftCLIService.java:404) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.thrift.ThriftCLIService.OpenSession(ThriftCLIService.java:299) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.thrift.TCLIService$Processor$OpenSession.getResult(TCLIService.java:1253) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.thrift.TCLIService$Processor$OpenSession.getResult(TCLIService.java:1238) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39) ~[libthrift-0.9.2.jar:0.9.2]
at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39) ~[libthrift-0.9.2.jar:0.9.2]
at org.apache.hive.service.auth.TSetIpAddressProcessor.process(TSetIpAddressProcessor.java:56) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:285) ~[libthrift-0.9.2.jar:0.9.2]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_131]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_131]
... 1 more
Caused by: java.lang.RuntimeException: java.lang.RuntimeException: java.io.IOException: No space left on device
at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:502) ~[?:?]
at org.apache.hive.service.cli.session.HiveSessionImpl.open(HiveSessionImpl.java:147) ~[hive-service-1.2.1.jar:1.2.1]
at sun.reflect.GeneratedMethodAccessor42.invoke(Unknown Source) ~[?:?]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_131]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_131]
at org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:78) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.session.HiveSessionProxy.access$000(HiveSessionProxy.java:36) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.session.HiveSessionProxy$1.run(HiveSessionProxy.java:63) ~[hive-service-1.2.1.jar:1.2.1]
at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_131]
at javax.security.auth.Subject.doAs(Subject.java:422) ~[?:1.8.0_131]
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1836) ~[hadoop-common-2.7.3.jar:?]
at org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:59) ~[hive-service-1.2.1.jar:1.2.1]
at com.sun.proxy.$Proxy20.open(Unknown Source) ~[?:?]
at org.apache.hive.service.cli.session.SessionManager.openSession(SessionManager.java:258) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.CLIService.openSessionWithImpersonation(CLIService.java:204) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.thrift.ThriftCLIService.getSessionHandle(ThriftCLIService.java:404) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.thrift.ThriftCLIService.OpenSession(ThriftCLIService.java:299) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.thrift.TCLIService$Processor$OpenSession.getResult(TCLIService.java:1253) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.thrift.TCLIService$Processor$OpenSession.getResult(TCLIService.java:1238) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39) ~[libthrift-0.9.2.jar:0.9.2]
at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39) ~[libthrift-0.9.2.jar:0.9.2]
at org.apache.hive.service.auth.TSetIpAddressProcessor.process(TSetIpAddressProcessor.java:56) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:285) ~[libthrift-0.9.2.jar:0.9.2]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_131]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_131]
... 1 more
Caused by: java.lang.RuntimeException: java.io.IOException: No space left on device
at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:495) ~[?:?]
at org.apache.hive.service.cli.session.HiveSessionImpl.open(HiveSessionImpl.java:147) ~[hive-service-1.2.1.jar:1.2.1]
at sun.reflect.GeneratedMethodAccessor42.invoke(Unknown Source) ~[?:?]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_131]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_131]
at org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:78) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.session.HiveSessionProxy.access$000(HiveSessionProxy.java:36) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.session.HiveSessionProxy$1.run(HiveSessionProxy.java:63) ~[hive-service-1.2.1.jar:1.2.1]
at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_131]
at javax.security.auth.Subject.doAs(Subject.java:422) ~[?:1.8.0_131]
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1836) ~[hadoop-common-2.7.3.jar:?]
at org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:59) ~[hive-service-1.2.1.jar:1.2.1]
at com.sun.proxy.$Proxy20.open(Unknown Source) ~[?:?]
at org.apache.hive.service.cli.session.SessionManager.openSession(SessionManager.java:258) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.CLIService.openSessionWithImpersonation(CLIService.java:204) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.thrift.ThriftCLIService.getSessionHandle(ThriftCLIService.java:404) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.thrift.ThriftCLIService.OpenSession(ThriftCLIService.java:299) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.thrift.TCLIService$Processor$OpenSession.getResult(TCLIService.java:1253) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.thrift.TCLIService$Processor$OpenSession.getResult(TCLIService.java:1238) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39) ~[libthrift-0.9.2.jar:0.9.2]
at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39) ~[libthrift-0.9.2.jar:0.9.2]
at org.apache.hive.service.auth.TSetIpAddressProcessor.process(TSetIpAddressProcessor.java:56) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:285) ~[libthrift-0.9.2.jar:0.9.2]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_131]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_131]
... 1 more
Caused by: java.io.IOException: No space left on device
at java.io.UnixFileSystem.createFileExclusively(Native Method) ~[?:1.8.0_131]
at java.io.File.createTempFile(File.java:2024) ~[?:1.8.0_131]
at org.apache.hadoop.hive.ql.session.SessionState.createTempFile(SessionState.java:798) ~[?:?]
at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:493) ~[?:?]
at org.apache.hive.service.cli.session.HiveSessionImpl.open(HiveSessionImpl.java:147) ~[hive-service-1.2.1.jar:1.2.1]
at sun.reflect.GeneratedMethodAccessor42.invoke(Unknown Source) ~[?:?]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_131]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_131]
at org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:78) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.session.HiveSessionProxy.access$000(HiveSessionProxy.java:36) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.session.HiveSessionProxy$1.run(HiveSessionProxy.java:63) ~[hive-service-1.2.1.jar:1.2.1]
at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_131]
at javax.security.auth.Subject.doAs(Subject.java:422) ~[?:1.8.0_131]
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1836) ~[hadoop-common-2.7.3.jar:?]
at org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:59) ~[hive-service-1.2.1.jar:1.2.1]
at com.sun.proxy.$Proxy20.open(Unknown Source) ~[?:?]
at org.apache.hive.service.cli.session.SessionManager.openSession(SessionManager.java:258) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.CLIService.openSessionWithImpersonation(CLIService.java:204) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.thrift.ThriftCLIService.getSessionHandle(ThriftCLIService.java:404) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.thrift.ThriftCLIService.OpenSession(ThriftCLIService.java:299) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.thrift.TCLIService$Processor$OpenSession.getResult(TCLIService.java:1253) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.hive.service.cli.thrift.TCLIService$Processor$OpenSession.getResult(TCLIService.java:1238) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39) ~[libthrift-0.9.2.jar:0.9.2]
at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39) ~[libthrift-0.9.2.jar:0.9.2]
at org.apache.hive.service.auth.TSetIpAddressProcessor.process(TSetIpAddressProcessor.java:56) ~[hive-service-1.2.1.jar:1.2.1]
at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:285) ~[libthrift-0.9.2.jar:0.9.2]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_131]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_131]
... 1 more
处理方案
确认服务器的磁盘使用情况
命令 df -h /opt
結果
[root@bigdata03-demo ~]# df -h /opt
文件系统 容量 已用 可用 已用% 挂载点
/dev/mapper/systemvg-optlv 33G 23G 11G 69% /opt
寻找存储最大的目录和文件
查询子目录的大小，子目录层级一级，命令du -h --max-depth=1 /opt
[root@bigdata03-demo ~]# du -h --max-depth=1 /opt
618M /opt/zookeeper
371M /opt/java
1.6G /opt/hive
8.3M /opt/sparkLocal
3.8G /opt/spark
76M /opt/promes
48K /opt/flinkLocal
172M /opt/monitor
22G /opt
确认删除文件
命令 rm -f 文件绝对路径
![[图解]SysML和EA建模住宅安全系统-10-活动图 入侵者应急响应](https://i-blog.csdnimg.cn/direct/a8a9e2f77d2b4faf9b79db80648c112c.png)
