Skip to content
This repository has been archived by the owner on Jun 9, 2022. It is now read-only.

Commit

Permalink
Issue orientechnologies#7998 has been fixed.
Browse files Browse the repository at this point in the history
  • Loading branch information
@andrii0lomakin
andrii0lomakin committed Mar 20, 2018
1 parent 19fc95c commit eb74721
Showing 1 changed file with 164 additions and 88 deletions.
252 changes: 164 additions & 88 deletions ...va/com/orientechnologies/orient/core/storage/impl/local/OStorageConfigurationSegment.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,62 +20,98 @@
package com.orientechnologies.orient.core.storage.impl.local;

import com.orientechnologies.common.exception.OException;
import com.orientechnologies.common.log.OLogManager;
import com.orientechnologies.common.serialization.types.OIntegerSerializer;
import com.orientechnologies.common.serialization.types.OLongSerializer;
import com.orientechnologies.orient.core.config.OGlobalConfiguration;
import com.orientechnologies.orient.core.config.OStorageConfiguration;
import com.orientechnologies.orient.core.config.OStorageFileConfiguration;
import com.orientechnologies.orient.core.exception.OSerializationException;
import com.orientechnologies.orient.core.exception.OStorageException;
import com.orientechnologies.orient.core.serialization.OBinaryProtocol;
import com.orientechnologies.orient.core.sql.parser.OInteger;
import com.orientechnologies.orient.core.storage.ORawBuffer;
import com.orientechnologies.orient.core.storage.fs.OFile;
import com.orientechnologies.orient.core.storage.impl.local.paginated.OLocalPaginatedStorage;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;

import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.util.Map;

/**
* Handles the database configuration in one big record.
* Handles the database configuration.
*/
@SuppressWarnings("serial")
@SuppressFBWarnings(value = "SE_TRANSIENT_FIELD_NOT_RESTORED")
public class OStorageConfigurationSegment extends OStorageConfiguration {
//This class uses "double write" pattern.
//Whenever we want to update configuration, first we write data in backup file and make fsync. Then we write the same data
//in primary file and make fsync. Then we remove backup file. So does not matter if we have error on any of this stages
//we always will have consistent storage configuration.
//Downside of this approach is the double overhead during write of configuration, but it was chosen to keep binary compatibility
//between versions.

/**
* Name of primary file
*/
private static final String NAME = "database.ocf";

/**
* Name of backup file which is used when we update storage configuration using double write pattern
*/
private static final String BACKUP_NAME = "database.ocf2";

private static final long serialVersionUID = 638874446554389034L;

private static final long ENCODING_FLAG_1 = 128975354756545L;
private static final long ENCODING_FLAG_2 = 587138568122547L;
private static final long ENCODING_FLAG_3 = 812587836547249L;

private static final int START_SIZE = 10000;
private final transient OSingleFileSegment segment;
private final String storageName;
private final String storagePath;

public OStorageConfigurationSegment(final OLocalPaginatedStorage storage) throws IOException {
super(storage, Charset.forName("UTF-8"));

public OStorageConfigurationSegment(final OLocalPaginatedStorage iStorage) throws IOException {
super(iStorage, Charset.forName("UTF-8"));
segment = new OSingleFileSegment((OLocalPaginatedStorage) storage,
new OStorageFileConfiguration(null, getDirectory() + "/database.ocf", "classic", fileTemplate.maxSize,
fileTemplate.fileIncrementSize));
this.storageName = storage.getName();
this.storagePath = storage.getStoragePath();
}

public void close() throws IOException {
super.close();
segment.close();
}

public void delete() throws IOException {
super.delete();
segment.delete();

clearConfigurationFiles();
}

/**
* Remove both backup and primary configuration files on delete
*
* @see #update()
*/
private void clearConfigurationFiles() {
final File file = new File(storagePath, NAME);
if (file.exists()) {
if (!file.delete()) {
OLogManager.instance().warn(this, "Can not delete database configuration file %s", file);
}
}

final File backupFile = new File(storagePath, BACKUP_NAME);
if (backupFile.exists()) {
if (!backupFile.delete()) {
OLogManager.instance().warn(this, "Can not delete backup of database configuration file %s", backupFile);
}
}
}

public void create() throws IOException {
segment.create(START_SIZE);
super.create();
clearConfigurationFiles();

final OFile f = segment.getFile();
if (OGlobalConfiguration.STORAGE_CONFIGURATION_SYNC_ON_UPDATE.getValueAsBoolean())
f.synch();
super.create();
}

@Override
Expand All @@ -85,61 +121,67 @@ public OStorageConfiguration load(final Map<String, Object> iProperties) throws

bindPropertiesToContext(iProperties);

if (segment.getFile().exists())
segment.open();
else {
segment.create(START_SIZE);
final File file = new File(storagePath, NAME);
final File backupFile = new File(storagePath, BACKUP_NAME);

// @COMPATIBILITY0.9.25
// CHECK FOR OLD VERSION OF DATABASE
final ORawBuffer rawRecord = storage.readRecord(CONFIG_RID, null, false, false, null).getResult();
if (rawRecord != null)
fromStream(rawRecord.buffer);
final File activeFile;

update();
return this;
if (file.exists()) {
activeFile = file;
} else if (backupFile.exists()) {
OLogManager.instance().warn(this,
"Seems like previous update to the storage '%s' configuration was finished incorrectly, reading from backup",
backupFile);
activeFile = backupFile;
} else {
throw new OStorageException("Can not find configuration file for storage " + storageName);
}

final int size = segment.getFile().readInt(0);
byte[] buffer = new byte[size];
segment.getFile().read(OBinaryProtocol.SIZE_INT, buffer, size);
final RandomAccessFile rnd = new RandomAccessFile(activeFile, "r");
try {
final int size = rnd.readInt();//size of string which contains database configuration
byte[] buffer = new byte[size];

if (segment.getFile().getFileSize() >= size + 2 * OIntegerSerializer.INT_SIZE + 3 * OLongSerializer.LONG_SIZE) {
//previous versions of database encode data using native charset encoding
//as result special characters in cluster or index names can be broken
//in new versions we use UTF-8 encoding to check which encoding is used
//encoding flag was added, we check it to know whether we should use UTF-8 or native encoding
rnd.readFully(buffer);

final long encodingFagOne = segment.getFile().readLong(OIntegerSerializer.INT_SIZE + size);
final long encodingFagTwo = segment.getFile().readLong(OIntegerSerializer.INT_SIZE + size + OLongSerializer.LONG_SIZE);
final long encodingFagThree = segment.getFile()
.readLong(OIntegerSerializer.INT_SIZE + size + 2 * OLongSerializer.LONG_SIZE);
if (rnd.length() >= size + 2 * OIntegerSerializer.INT_SIZE + 3 * OLongSerializer.LONG_SIZE) {
final long encodingFagOne = rnd.readLong();
final long encodingFagTwo = rnd.readLong();
final long encodingFagThree = rnd.readLong();

Charset streamEncoding = Charset.defaultCharset();
final Charset streamEncoding;

if (encodingFagOne == ENCODING_FLAG_1 && encodingFagTwo == ENCODING_FLAG_2 && encodingFagThree == ENCODING_FLAG_3) {
final byte[] utf8Encoded = "UTF-8".getBytes(Charset.forName("UTF-8"));
//those flags are added to distinguish between old version of configuration file and new one.
if (encodingFagOne == ENCODING_FLAG_1 && encodingFagTwo == ENCODING_FLAG_2 && encodingFagThree == ENCODING_FLAG_3) {
final byte[] utf8Encoded = "UTF-8".getBytes(Charset.forName("UTF-8"));

final int encodingNameLength = segment.getFile()
.readInt(OIntegerSerializer.INT_SIZE + size + 3 * OLongSerializer.LONG_SIZE);
final int encodingNameLength = rnd.readInt();

if (encodingNameLength == utf8Encoded.length) {
final byte[] binaryEncodingName = new byte[encodingNameLength];
segment.getFile().read(2 * OIntegerSerializer.INT_SIZE + size + 3 * OLongSerializer.LONG_SIZE, binaryEncodingName,
encodingNameLength);
final String encodingName = new String(binaryEncodingName, "UTF-8");
if (encodingNameLength == utf8Encoded.length) {
final byte[] binaryEncodingName = new byte[encodingNameLength];
rnd.readFully(binaryEncodingName);

if (encodingName.equals("UTF-8")) {
streamEncoding = Charset.forName("UTF-8");
final String encodingName = new String(binaryEncodingName, "UTF-8");

if (encodingName.equals("UTF-8")) {
streamEncoding = Charset.forName("UTF-8");
} else {
throw new OStorageException("Invalid format for configuration file " + activeFile + " for storage" + storageName);
}

fromStream(buffer, 0, buffer.length, streamEncoding);
} else {
throw new OStorageException("Invalid format for configuration file " + activeFile + " for storage" + storageName);
}
} else {
throw new OStorageException("Invalid format for configuration file " + activeFile + " for storage" + storageName);
}
} else {
fromStream(buffer, 0, buffer.length, Charset.defaultCharset());
}

fromStream(buffer, 0, buffer.length, streamEncoding);
} else {
fromStream(buffer, 0, buffer.length, Charset.defaultCharset());
} finally {
rnd.close();
}

} catch (IOException e) {
throw OException
.wrapException(new OSerializationException("Cannot load database configuration. The database seems corrupted"), e);
Expand All @@ -148,62 +190,96 @@ public OStorageConfiguration load(final Map<String, Object> iProperties) throws
}

@Override
public void lock() throws IOException {
public void lock() {
}

@Override
public void unlock() throws IOException {
public void unlock() {
}

@Override
public void update() throws OSerializationException {
try {
final OFile f = segment.getFile();
final Charset utf8 = Charset.forName("UTF-8");
final byte[] buffer = toStream(utf8);

if (!f.isOpen())
return;
final String encodingName = "UTF-8";
final byte[] binaryEncodingName = encodingName.getBytes(utf8);
//length of presentation of configuration + configuration + 3 utf-8 encoding flags + length of utf-8 encoding name +
//utf-8 encoding name
final int len = buffer.length + 2 * OBinaryProtocol.SIZE_INT + binaryEncodingName.length + 3 * OLongSerializer.LONG_SIZE;

final Charset utf8 = Charset.forName("UTF-8");
final byte[] buffer = toStream(utf8);
final ByteBuffer byteBuffer = ByteBuffer.allocate(len);
byteBuffer.putInt(buffer.length);
byteBuffer.put(buffer);

final String encodingName = "UTF-8";
final byte[] binaryEncodingName = encodingName.getBytes(utf8);
byteBuffer.putLong(ENCODING_FLAG_1);
byteBuffer.putLong(ENCODING_FLAG_2);
byteBuffer.putLong(ENCODING_FLAG_3);

//length of presentation of configuration + configuration + 3 utf-8 encoding flags + length of utf-8 encoding name +
//utf-8 encoding name
final int len = buffer.length + 2 * OBinaryProtocol.SIZE_INT + binaryEncodingName.length + 3 * OLongSerializer.LONG_SIZE;
byteBuffer.putInt(binaryEncodingName.length);
byteBuffer.put(binaryEncodingName);

if (len > f.getFileSize())
f.allocateSpace(len - f.getFileSize());
try {
final File storagePath = new File(this.storagePath);
if (!storagePath.exists()) {
if (!storagePath.mkdirs()) {
throw new OStorageException("Can not create directory " + storagePath + " of location of storage " + storageName);
}
}

f.writeInt(0, buffer.length);
f.write(OBinaryProtocol.SIZE_INT, buffer);
final File backupFile = new File(storagePath, BACKUP_NAME);
if (backupFile.exists()) {
if (!backupFile.delete()) {
throw new OStorageException("Can not delete backup file " + backupFile + " in storage " + storageName);
}
}

//indicator that stream is encoded using UTF-8 encoding
f.writeLong(OIntegerSerializer.INT_SIZE + buffer.length, ENCODING_FLAG_1);
f.writeLong(OIntegerSerializer.INT_SIZE + buffer.length + OLongSerializer.LONG_SIZE, ENCODING_FLAG_2);
f.writeLong(OIntegerSerializer.INT_SIZE + buffer.length + 2 * OLongSerializer.LONG_SIZE, ENCODING_FLAG_3);
RandomAccessFile rnd = new RandomAccessFile(backupFile, "rw");
try {
rnd.write(byteBuffer.array());

f.writeInt(OIntegerSerializer.INT_SIZE + buffer.length + 3 * OLongSerializer.LONG_SIZE, binaryEncodingName.length);
f.write(2 * OIntegerSerializer.INT_SIZE + buffer.length + 3 * OLongSerializer.LONG_SIZE, binaryEncodingName);
if (OGlobalConfiguration.STORAGE_CONFIGURATION_SYNC_ON_UPDATE.getValueAsBoolean()) {
rnd.getFD().sync();
}
} finally {
rnd.close();
}

if (OGlobalConfiguration.STORAGE_CONFIGURATION_SYNC_ON_UPDATE.getValueAsBoolean())
f.synch();
final File file = new File(storagePath, NAME);
if (file.exists()) {
if (!file.delete()) {
throw new OStorageException("Can not delete configuration file " + file + " in storage " + storageName);
}
}

rnd = new RandomAccessFile(file, "rw");
try {
rnd.write(byteBuffer.array());

if (OGlobalConfiguration.STORAGE_CONFIGURATION_SYNC_ON_UPDATE.getValueAsBoolean()) {
rnd.getFD().sync();
}
} finally {
rnd.close();
}

if (!backupFile.delete()) {
throw new OStorageException("Can not delete backup file " + backupFile + " in storage " + storageName);
}

} catch (Exception e) {
throw OException.wrapException(new OSerializationException("Error on update storage configuration"), e);
}
}

public void synch() throws IOException {
segment.getFile().synch();
public void synch() {
}

@Override
public void setSoftlyClosed(boolean softlyClosed) throws IOException {
public void setSoftlyClosed(boolean softlyClosed) {
}

public String getFileName() {
return segment.getFile().getName();
return NAME;
}
}

0 comments on commit eb74721

Please sign in to comment.