Skip to content

Commit

Permalink
fixed #62
Browse files Browse the repository at this point in the history
  • Loading branch information
@dsukhoroslov
dsukhoroslov committed Sep 5, 2016
1 parent fd02103 commit 7d0059e
Show file tree
Hide file tree
Showing 2 changed files with 121 additions and 97 deletions.
182 changes: 99 additions & 83 deletions ...r-hazelcast/src/main/java/com/bagri/xdm/cache/hazelcast/store/FileDocumentCacheStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
Expand All @@ -31,6 +30,7 @@

import com.bagri.common.util.FileUtils;
import com.bagri.xdm.api.XDMException;
import com.bagri.xdm.cache.api.DocumentManagement;
import com.bagri.xdm.cache.hazelcast.impl.DocumentManagementImpl;
import com.bagri.xdm.cache.hazelcast.impl.PopulationManagementImpl;
import com.bagri.xdm.cache.hazelcast.impl.SchemaRepositoryImpl;
Expand Down Expand Up @@ -117,6 +117,57 @@ private String getFullUri(String fileName) {
return dataPath + "/" + fileName;
}

@Override
public Set<DocumentKey> loadAllKeys() {
//if (true) {
// return Collections.emptySet();
//}

ensureRepository();
if (xdmRepo == null) {
logger.trace("loadAllKeys.enter; store is not ready yet, skipping population");
return null;
}

logger.trace("loadAllKeys.enter;");
Set<DocumentKey> docIds = popManager.getDocumentKeys();
if (docIds != null) {
if (logger.isTraceEnabled()) {
logger.trace("loadAllKeys.exit; returning from PopulationManager: {}", docIds);
} else {
logger.info("loadAllKeys.exit; returning keys from PopulationManager: {}", docIds.size());
}
return docIds;
}

Path root = Paths.get(dataPath);
Map<DocumentKey, String> uris = new HashMap<>();
try {
List<Path> files = new ArrayList<>();
processPathFiles(root, files);
DocumentKey docKey;
for (Path path: files) {
String uri = path.getFileName().toString();
int revision = 0;
do {
docKey = xdmRepo.getFactory().newDocumentKey(uri, revision, dvFirst);
revision++;
} while (uris.get(docKey) != null);
uris.put(docKey, uri);
}
docIds = new HashSet<>(uris.keySet());
} catch (IOException ex) {
logger.error("loadAllKeys.error;", ex);
}
popManager.setKeyMappings(uris);
if (logger.isTraceEnabled()) {
logger.trace("loadAllKeys.exit; got mappings: {}", uris);
} else {
logger.info("loadAllKeys.exit; returning keys: {}", docIds.size());
}
return docIds;
}

private Document loadDocument(DocumentKey docKey) {
String docUri = null;
Document doc = popManager.getDocument(docKey.getKey());
Expand Down Expand Up @@ -180,67 +231,7 @@ public Map<DocumentKey, Document> loadAll(Collection<DocumentKey> keys) {
return result;
}

@Override
public Set<DocumentKey> loadAllKeys() {
//if (true) {
// return Collections.emptySet();
//}

ensureRepository();
if (xdmRepo == null) {
logger.trace("loadAllKeys.enter; store is not ready yet, skipping population");
return null;
}

logger.trace("loadAllKeys.enter;");
Set<DocumentKey> docIds = popManager.getDocumentKeys();
if (docIds != null) {
if (logger.isTraceEnabled()) {
logger.trace("loadAllKeys.exit; returning from PopulationManager: {}", docIds);
} else {
logger.info("loadAllKeys.exit; returning keys from PopulationManager: {}", docIds.size());
}
return docIds;
}

Path root = Paths.get(dataPath);
Map<DocumentKey, String> uris = new HashMap<>();
try {
List<Path> files = new ArrayList<>();
processPathFiles(root, files);
Collections.sort(files);
DocumentKey docKey;
for (Path path: files) {
String uri = path.getFileName().toString();
int revision = 0;
do {
docKey = xdmRepo.getFactory().newDocumentKey(uri, revision, dvFirst);
revision++;
} while (uris.get(docKey) != null);
uris.put(docKey, uri);
}
docIds = new HashSet<>(uris.keySet());
} catch (IOException ex) {
logger.error("loadAllKeys.error;", ex);
}
popManager.setKeyMappings(uris);
if (logger.isTraceEnabled()) {
logger.trace("loadAllKeys.exit; got mappings: {}", uris);
} else {
logger.info("loadAllKeys.exit; returning keys: {}", docIds.size());
}
return docIds;
}

@Override
public void store(DocumentKey key, Document value) {
logger.trace("store.enter; key: {}; value: {}", key, value);
ensureRepository();
if (xdmRepo == null) {
logger.trace("store; not ready yet, skipping store");
return;
}

private Exception storeDocument(DocumentManagement docManager, DocumentKey key, Document value) {
String docUri = popManager.getKeyMapping(key);
if (docUri == null) {
// create a new document
Expand All @@ -253,28 +244,55 @@ public void store(DocumentKey key, Document value) {

String fullUri = getFullUri(docUri);
try {
DocumentManagementImpl docManager = (DocumentManagementImpl) xdmRepo.getDocumentManagement();
String xml = docManager.getDocumentAsString(key, null);
FileUtils.writeTextFile(fullUri, xml);
logger.trace("store.exit; stored as: {}; length: {}", fullUri, xml.length());
logger.trace("storeDocument.exit; stored as: {}; length: {}", fullUri, xml.length());
return null;
} catch (IOException | XDMException ex) {
return ex;
}
}

@Override
public void store(DocumentKey key, Document value) {
logger.trace("store.enter; key: {}; value: {}", key, value);
ensureRepository();
DocumentManagement docManager = (DocumentManagement) xdmRepo.getDocumentManagement();
Exception ex = storeDocument(docManager, key, value);
if (ex != null) {
logger.error("store.error; exception on store document: " + ex.getMessage(), ex);
// rethrow it ?
throw new RuntimeException(ex);
} else {
logger.trace("store.exit");
}
}

@Override
public void storeAll(Map<DocumentKey, Document> entries) {
logger.trace("storeAll.enter; entries: {}", entries.size());
ensureRepository();
int cnt = 0;
int err = 0;
Exception ex = null;
DocumentManagement docManager = (DocumentManagement) xdmRepo.getDocumentManagement();
for (Map.Entry<DocumentKey, Document> entry: entries.entrySet()) {
store(entry.getKey(), entry.getValue());
Exception e = storeDocument(docManager, entry.getKey(), entry.getValue());
if (e == null) {
cnt++;
} else {
err++;
ex = e;
}
}
if (err == 0) {
logger.trace("storeAll.exit; stored: {}; errors: {}", cnt, err);
} else {
logger.info("storeAll.exit; stored: {}; errors: {}", cnt, err);
throw new RuntimeException(ex);
}
logger.trace("storeAll.exit; stored: {}", entries.size());
}

@Override
public void delete(DocumentKey key) {
logger.trace("delete.enter; key: {}", key);

private boolean deleteDocument(DocumentKey key) {
boolean result = false;
String docUri = popManager.deleteKeyMapping(key);
if (docUri != null) {
Expand All @@ -283,9 +301,16 @@ public void delete(DocumentKey key) {
try {
result = Files.deleteIfExists(path);
} catch (IOException ex) {
logger.error("delete.error; path: " + path, ex);
logger.error("deleteDocument.error; path: " + path, ex);
}
}
return result;
}

@Override
public void delete(DocumentKey key) {
logger.trace("delete.enter; key: {}", key);
boolean result = deleteDocument(key);
logger.trace("delete.exit; deleted: {}", result);
}

Expand All @@ -294,17 +319,8 @@ public void deleteAll(Collection<DocumentKey> keys) {
logger.trace("deleteAll.enter; keys: {}", keys.size());
int deleted = 0;
for (DocumentKey key: keys) {
String docUri = popManager.deleteKeyMapping(key);
if (docUri != null) {
docUri = getFullUri(docUri);
Path path = Paths.get(docUri);
try {
if (Files.deleteIfExists(path)) {
deleted++;
}
} catch (IOException ex) {
logger.error("deleteAll.error; path: " + path, ex);
}
if (deleteDocument(key)) {
deleted++;
}
}
logger.trace("deleteAll.exit; deleted: {}", deleted);
Expand Down
36 changes: 22 additions & 14 deletions bagri-xdm/src/main/java/com/bagri/common/util/FileUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,21 @@

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.RandomAccessFile;
import java.io.Writer;
import java.net.URI;
import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Scanner;

/**
* A set of static utility methods regarding files
Expand Down Expand Up @@ -47,13 +51,16 @@ public static String readTextFile(String fileName) throws IOException {
* @throws IOException in case of read error
*/
public static String readTextFile(String fileName, String encoding) throws IOException {
Path path = Paths.get(fileName);
Charset cs = Charset.forName(encoding);
StringBuilder text = new StringBuilder();
try (Scanner scanner = new Scanner(path, encoding)){
while (scanner.hasNextLine()) {
text.append(scanner.nextLine()).append(EOL);
}
}
try (FileInputStream fis = new FileInputStream(fileName)) {
FileChannel ch = fis.getChannel();
MappedByteBuffer mbb = ch.map(FileChannel.MapMode.READ_ONLY, 0, ch.size());
while (mbb.hasRemaining()) {
CharBuffer cb = cs.decode(mbb);
text.append(cb.toString());
}
}
return text.toString();
}

Expand All @@ -65,11 +72,12 @@ public static String readTextFile(String fileName, String encoding) throws IOExc
* @throws IOException in case of write error
*/
public static void writeTextFile(String fileName, String content) throws IOException {

try (Writer writer = new BufferedWriter(
new OutputStreamWriter(new FileOutputStream(fileName), def_encoding))) {
writer.write(content);
}
byte[] bytes = content.getBytes();
try (RandomAccessFile raw = new RandomAccessFile(fileName, "rw")) {
FileChannel ch = raw.getChannel();
ByteBuffer buff = ch.map(FileChannel.MapMode.READ_WRITE, 0, bytes.length);
buff.put(bytes);
}
}

/**
Expand All @@ -80,7 +88,7 @@ public static void writeTextFile(String fileName, String content) throws IOExcep
* @throws IOException in case of write error
*/
public static void appendTextFile(String fileName, String content) throws IOException {

// TODO: do append via NIO
try (Writer writer = new BufferedWriter(
new OutputStreamWriter(new FileOutputStream(fileName, true), def_encoding))) {
writer.write(content);
Expand Down

0 comments on commit 7d0059e

Please sign in to comment.