在 MySQL / Java / Hibernate 中保存大字符串

Question

我正在尝试将 json 文档保存到 mysql 8 数据库中。这个 json 文档超过 1GB，所以列类型是 longtext 来处理额外的大小。

但是，每当我尝试将 json blob 保存为 java 大小时，它都会引发关于字符串太长的错误。我的代码：

package com.cpa.ipf.bo.forecast;

import com.cpa.ipf.bo.CostCentre;
import com.vladmihalcea.hibernate.type.json.JsonStringType;
import org.hibernate.annotations.Type;
import org.hibernate.annotations.TypeDef;

import javax.persistence.*;
import java.sql.Clob;
import java.time.LocalDateTime;
import java.util.Objects;

@Entity
public class ForecastMetadata {
    @Id
    @GeneratedValue(strategy = GenerationType.AUTO)
    private Long id;

    private Long userId;

    @ManyToOne
    private CostCentre costCentre;



    private String name;

    private Boolean viewableByAll;

    @Enumerated(EnumType.STRING)
    private ForecastStatus status;

    private LocalDateTime tsCreated;

    private LocalDateTime tsFinished;

    @Lob
    private Clob forecastBody;

    public ForecastMetadata(Long id, Long userId, CostCentre costCentre, String name, Boolean viewableByAll, ForecastStatus status, LocalDateTime tsCreated, LocalDateTime tsFinished, Clob forecastBody) {
        this.id = id;
        this.userId = userId;
        this.costCentre = costCentre;
        this.name = name;
        this.viewableByAll = viewableByAll;
        this.status = status;
        this.tsCreated = tsCreated;
        this.tsFinished = tsFinished;
        this.forecastBody = forecastBody;
    }

    public ForecastMetadata() {}

    public Long getId() {
        return id;
    }

    public void setId(Long id) {
        this.id = id;
    }

    public Long getUserId() {
        return userId;
    }

    public void setUserId(Long userId) {
        this.userId = userId;
    }

    public CostCentre getCostCentre() {
        return costCentre;
    }

    public void setCostCentre(CostCentre costCentre) {
        this.costCentre = costCentre;
    }

    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }

    public Boolean getViewableByAll() {
        return viewableByAll;
    }

    public void setViewableByAll(Boolean viewableByAll) {
        this.viewableByAll = viewableByAll;
    }

    public ForecastStatus getStatus() {
        return status;
    }

    public void setStatus(ForecastStatus status) {
        this.status = status;
    }

    public LocalDateTime getTsCreated() {
        return tsCreated;
    }

    public void setTsCreated(LocalDateTime tsCreated) {
        this.tsCreated = tsCreated;
    }

    public LocalDateTime getTsFinished() {
        return tsFinished;
    }

    public void setTsFinished(LocalDateTime tsFinished) {
        this.tsFinished = tsFinished;
    }

    public Clob getForecastBody() {
        return forecastBody;
    }

    public void setForecastBody(Clob forecastBody) {
        this.forecastBody = forecastBody;
    }

    @Override
    public boolean equals(Object o) {
        if (this == o) return true;
        if (o == null || getClass() != o.getClass()) return false;
        ForecastMetadata that = (ForecastMetadata) o;
        return Objects.equals(id, that.id) &&
                Objects.equals(userId, that.userId) &&
                Objects.equals(costCentre, that.costCentre) &&
                Objects.equals(name, that.name) &&
                Objects.equals(viewableByAll, that.viewableByAll) &&
                status == that.status &&
                Objects.equals(tsCreated, that.tsCreated) &&
                Objects.equals(tsFinished, that.tsFinished) &&
                Objects.equals(forecastBody, that.forecastBody);
    }

    @Override
    public int hashCode() {
        return Objects.hash(id, userId, costCentre, name, viewableByAll, status, tsCreated, tsFinished, forecastBody);
    }

    @Override
    public String toString() {
        return "ForecastMetadata{" +
                "id=" + id +
                ", userId=" + userId +
                ", costCentre=" + costCentre +
                ", name='" + name + '\'' +
                ", viewableByAll=" + viewableByAll +
                ", status=" + status +
                ", tsCreated=" + tsCreated +
                ", tsFinished=" + tsFinished +
                ", forecastBody='" + forecastBody + '\'' +
                '}';
    }
}

错误：

java.lang.OutOfMemoryError: Requested array size exceeds VM limit
        at java.lang.AbstractStringBuilder.<init>(AbstractStringBuilder.java:68) ~[na:1.8.0_252]
        at java.lang.StringBuilder.<init>(StringBuilder.java:101) ~[na:1.8.0_252]
        at com.mysql.cj.ClientPreparedQueryBindings.setString(ClientPreparedQueryBindings.java:628) ~[mysql-connector-java-8.0.20.jar!/:8.0.20]
        at com.mysql.cj.ClientPreparedQueryBindings.setCharacterStream(ClientPreparedQueryBindings.java:335) ~[mysql-connector-java-8.0.20.jar!/:8.0.20]
        at com.mysql.cj.ClientPreparedQueryBindings.setCharacterStream(ClientPreparedQueryBindings.java:366) ~[mysql-connector-java-8.0.20.jar!/:8.0.20]
        at com.mysql.cj.jdbc.ClientPreparedStatement.setCharacterStream(ClientPreparedStatement.java:1519) ~[mysql-connector-java-8.0.20.jar!/:8.0.20]
        at org.hibernate.type.descriptor.sql.ClobTypeDescriptor$4$1.doBind(ClobTypeDescriptor.java:124) ~[hibernate-core-4.3.11.Final.jar!/:4.3.11.Final]
        at org.hibernate.type.descriptor.sql.BasicBinder.bind(BasicBinder.java:90) ~[hibernate-core-4.3.11.Final.jar!/:4.3.11.Final]
        at org.hibernate.type.AbstractStandardBasicType.nullSafeSet(AbstractStandardBasicType.java:286) ~[hibernate-core-4.3.11.Final.jar!/:4.3.11.Final]
        at org.hibernate.type.AbstractStandardBasicType.nullSafeSet(AbstractStandardBasicType.java:281) ~[hibernate-core-4.3.11.Final.jar!/:4.3.11.Final]
        at org.hibernate.type.AbstractSingleColumnStandardBasicType.nullSafeSet(AbstractSingleColumnStandardBasicType.java:56) ~[hibernate-core-4.3.11.Final.jar!/:4.3.11.Final]
        at org.hibernate.persister.entity.AbstractEntityPersister.dehydrate(AbstractEntityPersister.java:2843) ~[hibernate-core-4.3.11.Final.jar!/:4.3.11.Final]
        at org.hibernate.persister.entity.AbstractEntityPersister.dehydrate(AbstractEntityPersister.java:2818) ~[hibernate-core-4.3.11.Final.jar!/:4.3.11.Final]
        at org.hibernate.persister.entity.AbstractEntityPersister$4.bindValues(AbstractEntityPersister.java:3025) ~[hibernate-core-4.3.11.Final.jar!/:4.3.11.Final]
        at org.hibernate.id.insert.AbstractReturningDelegate.performInsert(AbstractReturningDelegate.java:57) ~[hibernate-core-4.3.11.Final.jar!/:4.3.11.Final]
        at org.hibernate.persister.entity.AbstractEntityPersister.insert(AbstractEntityPersister.java:3032) ~[hibernate-core-4.3.11.Final.jar!/:4.3.11.Final]
        at org.hibernate.persister.entity.AbstractEntityPersister.insert(AbstractEntityPersister.java:3558) ~[hibernate-core-4.3.11.Final.jar!/:4.3.11.Final]
        at org.hibernate.action.internal.EntityIdentityInsertAction.execute(EntityIdentityInsertAction.java:98) ~[hibernate-core-4.3.11.Final.jar!/:4.3.11.Final]
        at org.hibernate.engine.spi.ActionQueue.execute(ActionQueue.java:492) ~[hibernate-core-4.3.11.Final.jar!/:4.3.11.Final]
        at org.hibernate.engine.spi.ActionQueue.addResolvedEntityInsertAction(ActionQueue.java:197) ~[hibernate-core-4.3.11.Final.jar!/:4.3.11.Final]
        at org.hibernate.engine.spi.ActionQueue.addInsertAction(ActionQueue.java:181) ~[hibernate-core-4.3.11.Final.jar!/:4.3.11.Final]
        at org.hibernate.engine.spi.ActionQueue.addAction(ActionQueue.java:216) ~[hibernate-core-4.3.11.Final.jar!/:4.3.11.Final]
        at org.hibernate.event.internal.AbstractSaveEventListener.addInsertAction(AbstractSaveEventListener.java:334) ~[hibernate-core-4.3.11.Final.jar!/:4.3.11.Final]
        at org.hibernate.event.internal.AbstractSaveEventListener.performSaveOrReplicate(AbstractSaveEventListener.java:289) ~[hibernate-core-4.3.11.Final.jar!/:4.3.11.Final]
        at org.hibernate.event.internal.AbstractSaveEventListener.performSave(AbstractSaveEventListener.java:195) ~[hibernate-core-4.3.11.Final.jar!/:4.3.11.Final]
        at org.hibernate.event.internal.AbstractSaveEventListener.saveWithGeneratedId(AbstractSaveEventListener.java:126) ~[hibernate-core-4.3.11.Final.jar!/:4.3.11.Final]
        at org.hibernate.jpa.event.internal.core.JpaPersistEventListener.saveWithGeneratedId(JpaPersistEventListener.java:84) ~[hibernate-entitymanager-4.3.11.Final.jar!/:4.3.11.Final]
        at org.hibernate.event.internal.DefaultPersistEventListener.entityIsTransient(DefaultPersistEventListener.java:206) ~[hibernate-core-4.3.11.Final.jar!/:4.3.11.Final]
        at org.hibernate.event.internal.DefaultPersistEventListener.onPersist(DefaultPersistEventListener.java:149) ~[hibernate-core-4.3.11.Final.jar!/:4.3.11.Final]
        at org.hibernate.event.internal.DefaultPersistEventListener.onPersist(DefaultPersistEventListener.java:75) ~[hibernate-core-4.3.11.Final.jar!/:4.3.11.Final]
        at org.hibernate.internal.SessionImpl.firePersist(SessionImpl.java:811) ~[hibernate-core-4.3.11.Final.jar!/:4.3.11.Final]
        at org.hibernate.internal.SessionImpl.persist(SessionImpl.java:784) ~[hibernate-core-4.3.11.Final.jar!/:4.3.11.Final]

有没有办法解决这个问题？

Answer 1

我看到很多cmets说你应该增加堆大小，但是错误信息

请求的数组大小超过虚拟机限制

实际上意味着尝试创建一个数组，其 length 将超过特定于平台的最大可能数组 length（通常在 Integer.MAX_VALUE 左右）。增加堆大小不会有帮助。这不是对内存中数组的大小的限制，而是对数组可以具有的最大有效索引。

现在，让我们看看导致问题的代码：

StringBuilder buf = new StringBuilder((int) (x.length() * 1.1));

在上面的代码中，x 将您的Clob 的内容表示为String。 StringBuilder 的构造函数委托给AbstractStringBuilder 构造函数，如下所示：

AbstractStringBuilder(int capacity) {
        if (COMPACT_STRINGS) {
            value = new byte[capacity];
            coder = LATIN1;
        } else {
            value = StringUTF16.newBytesFor(capacity);
            coder = UTF16;
        }
    }

这里我们看到了问题，简单明了：代表您的CLOB 的String 的大小必须非常危险地接近最大数组长度限制，以至于其大小超过限制的 1.1 倍。

现在，让我们看看我们是否可以阻止 MySQL 连接器尝试初始化这样一个数组（请注意，我们还应该针对 MySQL 连接器不初始化 x，因为 String 在内部也表示为一个数组，并且您的数据已经危险地接近数组大小限制）。将堆栈向上移动一点到ClientPreparedQueryBindings.setCharacterStream()，我们得到：

boolean useLength = this.useStreamLengthsInPrepStmts.getValue();
String forcedEncoding = this.session.getPropertySet().getStringProperty(PropertyKey.clobCharacterEncoding).getStringValue();
if (useLength && (length != -1)) {
    c = new char[length];
    int numCharsRead = Util.readFully(reader, c, length); // blocks until all read
    if (forcedEncoding == null) {
        setString(parameterIndex, new String(c, 0, numCharsRead));
    } else {
        setBytes(parameterIndex, StringUtils.getBytes(new String(c, 0, numCharsRead), forcedEncoding));
    }
} else {
    c = new char[4096];
    StringBuilder buf = new StringBuilder();
    while ((len = reader.read(c)) != -1) {
        buf.append(c, 0, len);
    }
    if (forcedEncoding == null) {
        setString(parameterIndex, buf.toString());
    } else {
        setBytes(parameterIndex, StringUtils.getBytes(buf.toString(), forcedEncoding));
    }
}

看代码，好像你运气不好。您可以尝试强制执行跳转到 if 语句的其他分支之一（使用 MySql 连接属性），但无论您走到哪里，最终都会创建一个与输入大小相同的大数组。

您可能最好使用BLOB，因为setBinaryStream() 的实现方式如下：

public void setBinaryStream(int parameterIndex, InputStream x, int length) {
        if (x == null) {
            setNull(parameterIndex);
        } else {
            this.bindValues[parameterIndex].setNull(false);
            this.bindValues[parameterIndex].setIsStream(true);
            this.bindValues[parameterIndex].setMysqlType(MysqlType.BLOB); // TODO use length to find the right BLOB type
            this.bindValues[parameterIndex].setStreamValue(x, length);
        }
    }

...但不幸的是，您需要自己实现 TODO，才能获得 LONGBLOB 而不是 BLOB。我想这一切都归结为您非常想使用 JDBC 驱动程序将 2GB 大小的 blob 放入 MySQL，这显然不适合这项工作。