【发布时间】：2015-03-18 22:03:00
【问题描述】：
我正在尝试使用 Jena 的读取方法来读取大型数据集（超过 1 gb），但我收到内存不足错误。我尝试将 tomcat heapsize（-Xmx 参数）增加到 2048，这也是 eclipse.ini 文件中的相同参数。但是我无法获得有效的解决方案 我愿意接受有关如何处理大型数据集的任何建议，因为我会将数据集解析为哈希图并在网页上显示内容。
控制台错误如下：
Exception in thread "http-bio-8080-AsyncTimeout" java.lang.OutOfMemoryError: GC overhead limit exceeded
at java.util.concurrent.ConcurrentLinkedQueue.iterator(ConcurrentLinkedQueue.java:667)
at org.apache.tomcat.util.net.JIoEndpoint$AsyncTimeout.run(JIoEndpoint.java:157)
at java.lang.Thread.run(Thread.java:745)
Exception in thread "http-bio-8080-exec-6" java.lang.OutOfMemoryError: GC overhead limit exceeded
at java.util.concurrent.CopyOnWriteArrayList.iterator(CopyOnWriteArrayList.java:959)
at com.hp.hpl.jena.graph.impl.SimpleEventManager.notifyAddTriple(SimpleEventManager.java:91)
at com.hp.hpl.jena.graph.impl.GraphBase.notifyAdd(GraphBase.java:124)
at com.hp.hpl.jena.graph.impl.GraphBase.add(GraphBase.java:203)
at org.apache.jena.riot.system.StreamRDFLib$ParserOutputGraph.triple(StreamRDFLib.java:165)
at org.apache.jena.riot.lang.LangNTriples.runParser(LangNTriples.java:56)
at org.apache.jena.riot.lang.LangBase.parse(LangBase.java:42)
at org.apache.jena.riot.RDFParserRegistry$ReaderRIOTLang.read(RDFParserRegistry.java:182)
at org.apache.jena.riot.RDFDataMgr.process(RDFDataMgr.java:906)
at org.apache.jena.riot.RDFDataMgr.read(RDFDataMgr.java:257)
at org.apache.jena.riot.RDFDataMgr.read(RDFDataMgr.java:243)
at org.apache.jena.riot.adapters.RDFReaderRIOT_Web.read(RDFReaderRIOT_Web.java:96)
at com.hp.hpl.jena.rdf.model.impl.ModelCom.read(ModelCom.java:235)
at com.packages.rdf.FileAnalyse.GetFileComponents(FileAnalyse.java:77)
at com.packages.servlets.CreatePatternServlet.GetStatements(CreatePatternServlet.java:96)
at com.packages.servlets.CreatePatternServlet.doPost(CreatePatternServlet.java:68)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:646)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:727)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:220)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:501)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:171)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:950)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:408)
at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1070)
Exception in thread "ContainerBackgroundProcessor[StandardEngine[Catalina]]" java.lang.OutOfMemoryError: GC overhead limit exceeded
at org.apache.naming.resources.FileDirContext.file(FileDirContext.java:765)
at org.apache.naming.resources.FileDirContext.doGetAttributes(FileDirContext.java:398)
at org.apache.naming.resources.BaseDirContext.getAttributes(BaseDirContext.java:1137)
at org.apache.naming.resources.BaseDirContext.getAttributes(BaseDirContext.java:1090)
at org.apache.naming.resources.ProxyDirContext.getAttributes(ProxyDirContext.java:882)
at org.apache.catalina.loader.WebappClassLoader.modified(WebappClassLoader.java:1026)
at org.apache.catalina.loader.WebappLoader.modified(WebappLoader.java:500)
at org.apache.catalina.loader.WebappLoader.backgroundProcess(WebappLoader.java:420)
at org.apache.catalina.core.ContainerBase.backgroundProcess(ContainerBase.java:1345)
at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.processChildren(ContainerBase.java:1546)
at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.processChildren(ContainerBase.java:1556)
at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.processChildren(ContainerBase.java:1556)
at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.run(ContainerBase.java:1524)
at java.lang.Thread.run(Thread.java:745)
Exception in thread "http-bio-8080-exec-6" java.lang.OutOfMemoryError: GC overhead limit exceeded
at org.apache.jena.riot.tokens.TokenizerText.parseToken(TokenizerText.java:170)
at org.apache.jena.riot.tokens.TokenizerText.hasNext(TokenizerText.java:86)
at org.apache.jena.atlas.iterator.PeekIterator.fill(PeekIterator.java:50)
at org.apache.jena.atlas.iterator.PeekIterator.next(PeekIterator.java:92)
at org.apache.jena.riot.lang.LangEngine.nextToken(LangEngine.java:99)
at org.apache.jena.riot.lang.LangNTriples.parseOne(LangNTriples.java:71)
at org.apache.jena.riot.lang.LangNTriples.runParser(LangNTriples.java:54)
at org.apache.jena.riot.lang.LangBase.parse(LangBase.java:42)
at org.apache.jena.riot.RDFParserRegistry$ReaderRIOTLang.read(RDFParserRegistry.java:182)
at org.apache.jena.riot.RDFDataMgr.process(RDFDataMgr.java:906)
at org.apache.jena.riot.RDFDataMgr.read(RDFDataMgr.java:257)
at org.apache.jena.riot.RDFDataMgr.read(RDFDataMgr.java:243)
at org.apache.jena.riot.adapters.RDFReaderRIOT_Web.read(RDFReaderRIOT_Web.java:96)
at com.hp.hpl.jena.rdf.model.impl.ModelCom.read(ModelCom.java:235)
at com.packages.rdf.FileAnalyse.GetFileComponents(FileAnalyse.java:77)
at com.packages.servlets.CreatePatternServlet.GetStatements(CreatePatternServlet.java:96)
at com.packages.servlets.CreatePatternServlet.doPost(CreatePatternServlet.java:68)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:646)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:727)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:220)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:501)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:171)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:950)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:408)
【问题讨论】：
-
仅仅因为您的文件是 1 GB 并不意味着您只会用完 1 GB 的内存 - 由于各种原因，它通常会更多。您可以尝试将堆大小提高到更高的值，例如 8 GB 或 16 GB？ （顺便说一句，你不必写“2048M” - 你可以写“2G”，它的数量是一样的）
-
我尝试了 4g，但对于这个应用程序，我可能需要分析 10gb 的文件，所以我不知道它是否足够
-
那么您可能必须找到一个不会尝试将整个文件一次加载到内存中的库。恐怕我不知道有什么事情会在我脑海中浮现。但是，您的分析是否可以在多个较小的文件上运行，然后放在一起？如果是这样，您可以将巨大的文件分解成更小的文件。
标签： java eclipse garbage-collection apache-jena