Skip to content
This repository has been archived by the owner on Nov 19, 2020. It is now read-only.

Commit

Permalink
NEXUS-8226: limit reconciliation to checksum attributes modified sinc…
Browse files Browse the repository at this point in the history 
…e a given date
  • Loading branch information
@mcculls
mcculls committed Mar 11, 2015
1 parent ef1b443 commit 63e389d
Show file tree
Hide file tree
Showing 4 changed files with 133 additions and 25 deletions.
82 changes: 70 additions & 12 deletions components/nexus-core/src/main/java/org/sonatype/nexus/proxy/item/ChecksumReconciler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
*/
package org.sonatype.nexus.proxy.item;

import java.io.File;
import java.io.InputStream;
import java.security.DigestInputStream;
import java.security.MessageDigest;
Expand All @@ -26,6 +27,7 @@
import org.sonatype.nexus.proxy.attributes.inspectors.DigestCalculatingInspector;
import org.sonatype.nexus.proxy.maven.MUtils;
import org.sonatype.nexus.proxy.repository.Repository;
import org.sonatype.nexus.proxy.storage.local.fs.DefaultFSLocalRepositoryStorage;
import org.sonatype.nexus.proxy.walker.AbstractFileWalkerProcessor;
import org.sonatype.nexus.proxy.walker.DefaultWalkerContext;
import org.sonatype.nexus.proxy.walker.Walker;
Expand All @@ -35,6 +37,14 @@
import org.sonatype.sisu.goodies.common.ComponentSupport;

import com.google.common.io.ByteStreams;
import org.apache.commons.lang.StringUtils;

import static org.sonatype.nexus.proxy.attributes.inspectors.DigestCalculatingInspector.DIGEST_MD5_KEY;
import static org.sonatype.nexus.proxy.attributes.inspectors.DigestCalculatingInspector.DIGEST_SHA1_KEY;
import static org.sonatype.nexus.proxy.maven.ChecksumContentValidator.ATTR_REMOTE_MD5;
import static org.sonatype.nexus.proxy.maven.ChecksumContentValidator.ATTR_REMOTE_SHA1;
import static org.sonatype.nexus.proxy.maven.ChecksumContentValidator.SUFFIX_MD5;
import static org.sonatype.nexus.proxy.maven.ChecksumContentValidator.SUFFIX_SHA1;

/**
* Reconciles any item attribute checksums affected by NEXUS-8178.
Expand All @@ -53,6 +63,10 @@ public class ChecksumReconciler

private final MessageDigest md5;

private File attributesBaseDir;

private long modifiedSinceMillis;

@Inject
public ChecksumReconciler(final Walker walker, final DigestCalculatingInspector digestCalculatingInspector)
throws NoSuchAlgorithmException
Expand All @@ -68,10 +82,18 @@ public ChecksumReconciler(final Walker walker, final DigestCalculatingInspector
/**
* Walks the selected sub-tree in the given repository attempting to reconcile their item attribute checksums.
*/
public void reconcileChecksums(final Repository repo, final ResourceStoreRequest request) {
public void reconcileChecksums(final Repository repo, final ResourceStoreRequest request, final long sinceMillis) {
final WalkerContext context = new DefaultWalkerContext(repo, request);

final String repositoryId = repo.getId();

attributesBaseDir = getAttributesBaseDir(repo);
modifiedSinceMillis = sinceMillis;

if (attributesBaseDir == null) {
return; // no point walking repository with no local storage
}

context.getProcessors().add(new AbstractFileWalkerProcessor()
{
@Override
Expand All @@ -97,17 +119,21 @@ protected void processFileItem(final WalkerContext ctx, final StorageFileItem it
* Checks the checksum cached in the item's attributes to see if it needs reconciling with the stored content.
*/
void reconcileItemChecksum(final Repository repo, final StorageFileItem item) throws Exception {
final String itemPath = item.getPath();
if (!itemPath.endsWith(".sha1") && !itemPath.endsWith(".md5")) {
final String attributeSHA1 = item.getRepositoryItemAttributes().get(DigestCalculatingInspector.DIGEST_SHA1_KEY);
if (attributeSHA1 != null) {

final String localSHA1 = item.getRepositoryItemAttributes().get(DIGEST_SHA1_KEY);
final String remoteSHA1 = item.getRepositoryItemAttributes().get(ATTR_REMOTE_SHA1);

if (localSHA1 != null || remoteSHA1 != null) {

final String itemPath = item.getPath();
if (shouldAttemptReconciliation(itemPath)) {

sha1.reset();
md5.reset();

// look for checksums affected by the link persister pre-fetching the first 8 bytes (see NEXUS-8178)
try (final InputStream is = new DigestInputStream(new DigestInputStream( //
item.getContentLocator().getContent(), sha1), md5)) {
try (final InputStream is = new DigestInputStream(new DigestInputStream(item.getContentLocator().getContent(),
sha1), md5)) {

final byte[] buf = new byte[8];
ByteStreams.read(is, buf, 0, 8);
Expand All @@ -119,11 +145,16 @@ void reconcileItemChecksum(final Repository repo, final StorageFileItem item) th

final String affectedSHA1 = DigesterUtils.getDigestAsString(sha1.digest());

if (attributeSHA1.equals(affectedSHA1)) {
if ((localSHA1 != null && localSHA1.equals(affectedSHA1))
|| (remoteSHA1 != null && remoteSHA1.equals(affectedSHA1))) {

log.info("Reconciling attribute checksums for {}", item);

final RepositoryItemUid uid = item.getRepositoryItemUid();
uid.getLock().lock(Action.update);
try {
item.getRepositoryItemAttributes().remove(ATTR_REMOTE_SHA1);
item.getRepositoryItemAttributes().remove(ATTR_REMOTE_MD5);
digestCalculatingInspector.processStorageItem(item);
repo.getAttributesHandler().storeAttributes(item);
}
Expand All @@ -132,15 +163,42 @@ void reconcileItemChecksum(final Repository repo, final StorageFileItem item) th
}
}

reconcileChecksumFile(repo, itemPath + ".sha1", affectedSHA1,
item.getRepositoryItemAttributes().get(DigestCalculatingInspector.DIGEST_SHA1_KEY));
reconcileChecksumFile(repo, itemPath + SUFFIX_SHA1, affectedSHA1,
item.getRepositoryItemAttributes().get(DIGEST_SHA1_KEY));

final String affectedMD5 = DigesterUtils.getDigestAsString(md5.digest());

reconcileChecksumFile(repo, itemPath + ".md5", affectedMD5,
item.getRepositoryItemAttributes().get(DigestCalculatingInspector.DIGEST_MD5_KEY));
reconcileChecksumFile(repo, itemPath + SUFFIX_MD5, affectedMD5,
item.getRepositoryItemAttributes().get(DIGEST_MD5_KEY));
}
}
}

/**
* Determines the base directory of local attribute storage.
*/
private File getAttributesBaseDir(final Repository repo) {
try {
if (repo.getLocalStorage() instanceof DefaultFSLocalRepositoryStorage) {
final File baseDir = ((DefaultFSLocalRepositoryStorage) repo.getLocalStorage()).getBaseDir(repo, null);
return new File(new File(baseDir, ".nexus"), "attributes");
}
}
catch (final Exception e) {
log.warn("Problem finding local storage for {}", repo, e);
}
return null;
}

/**
* Should we attempt checksum reconciliation for the given path?
*/
private boolean shouldAttemptReconciliation(final String itemPath) {
if (itemPath == null || itemPath.endsWith(SUFFIX_SHA1) || itemPath.endsWith(SUFFIX_MD5)) {
return false; // ignore associated checksum files, we'll fix them when we process the main item
}
final File attributesFile = new File(attributesBaseDir, StringUtils.strip(itemPath, "/"));
return attributesFile.lastModified() >= modifiedSinceMillis;
}

/**
Expand Down
36 changes: 25 additions & 11 deletions components/nexus-core/src/main/java/org/sonatype/nexus/tasks/ReconcileChecksumsTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,9 @@
import org.sonatype.nexus.scheduling.AbstractNexusRepositoriesPathAwareTask;
import org.sonatype.nexus.tasks.descriptors.ReconcileChecksumsTaskDescriptor;
import org.sonatype.nexus.util.LinearNumberSequence;
import org.sonatype.nexus.util.SystemPropertiesHelper;

import org.apache.commons.lang.StringUtils;
import org.joda.time.DateTime;

import static com.google.common.base.Preconditions.checkNotNull;

Expand All @@ -45,13 +47,6 @@ public class ReconcileChecksumsTask
*/
public static final String ACTION = "RECONCILECHECKSUMS";

/**
* System property used to control throttling
*/
public static final String MAX_RATE_KEY = ReconcileChecksumsTask.class.getName() + ".maxRate";

private static final int MAX_RATE = SystemPropertiesHelper.getInteger(MAX_RATE_KEY, 100);

private final ChecksumReconciler checksumReconciler;

@Inject
Expand All @@ -69,6 +64,24 @@ protected String getRepositoryPathFieldId() {
return ReconcileChecksumsTaskDescriptor.RESOURCE_STORE_PATH_FIELD_ID;
}

public long getModifiedSinceMillis() {
final String value = getParameter(ReconcileChecksumsTaskDescriptor.MODIFIED_SINCE_DATE_ID);
return StringUtils.isNotBlank(value) ? DateTime.parse(value).getMillis() : -1;
}

public void setModifiedSinceDate(final String modifiedSinceDate) {
addParameter(ReconcileChecksumsTaskDescriptor.MODIFIED_SINCE_DATE_ID, modifiedSinceDate);
}

public int getWalkingLimitTps() {
final String value = getParameter(ReconcileChecksumsTaskDescriptor.WALKING_LIMIT_TPS_FIELD_ID);
return StringUtils.isNotBlank(value) ? Integer.parseInt(value) : -1;
}

public void setWalkingLimitTps(final int walkingLimitTps) {
addParameter(ReconcileChecksumsTaskDescriptor.WALKING_LIMIT_TPS_FIELD_ID, Integer.toString(walkingLimitTps));
}

@Override
public Object doRun() throws Exception {
final List<Repository> targetRepositories = new ArrayList<>();
Expand All @@ -95,13 +108,14 @@ public Object doRun() throws Exception {

final ResourceStoreRequest request = new ResourceStoreRequest(getResourceStorePath(), true, false);

if (MAX_RATE > 0) {
final int limitTps = getWalkingLimitTps();
if (limitTps > 0) {
request.getRequestContext().put(WalkerThrottleController.CONTEXT_KEY,
new FixedRateWalkerThrottleController(MAX_RATE, new LinearNumberSequence(0, 1, 1, 0)));
new FixedRateWalkerThrottleController(limitTps, new LinearNumberSequence(0, 1, 1, 0)));
}

for (final Repository repo : targetRepositories) {
checksumReconciler.reconcileChecksums(repo, request);
checksumReconciler.reconcileChecksums(repo, request, getModifiedSinceMillis());
}

return null;
Expand Down
18 changes: 17 additions & 1 deletion .../src/main/java/org/sonatype/nexus/tasks/descriptors/ReconcileChecksumsTaskDescriptor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import javax.inject.Singleton;

import org.sonatype.nexus.formfields.FormField;
import org.sonatype.nexus.formfields.NumberTextFormField;
import org.sonatype.nexus.formfields.RepoOrGroupComboFormField;
import org.sonatype.nexus.formfields.StringTextFormField;

Expand All @@ -36,6 +37,10 @@ public class ReconcileChecksumsTaskDescriptor

public static final String RESOURCE_STORE_PATH_FIELD_ID = "resourceStorePath";

public static final String MODIFIED_SINCE_DATE_ID = "modifiedSinceDate";

public static final String WALKING_LIMIT_TPS_FIELD_ID = "walkingLimitTps";

private final RepoOrGroupComboFormField repoField = new RepoOrGroupComboFormField(REPO_OR_GROUP_FIELD_ID,
FormField.MANDATORY);

Expand All @@ -44,6 +49,16 @@ public class ReconcileChecksumsTaskDescriptor
"Enter a repository path to run the task in recursively (ie. \"/\" for root or \"/org/apache\").",
FormField.OPTIONAL);

private final StringTextFormField modifiedSinceDateField = new StringTextFormField(MODIFIED_SINCE_DATE_ID,
"Modified since (yyyy-MM-dd)",
"Enter a date to limit reconciliation to those checksum attributes modified since the given date.",
FormField.OPTIONAL).withInitialValue("2015-01-01");

private final NumberTextFormField walkingLimitTpsField = new NumberTextFormField(WALKING_LIMIT_TPS_FIELD_ID,
"Walking limit (tps)",
"Set the walking limit to reduce the overhead of this task at the expense of it taking more time.",
FormField.OPTIONAL).withInitialValue(100);

public String getId() {
return ID;
}
Expand All @@ -57,8 +72,9 @@ public List<FormField> formFields() {
final List<FormField> fields = new ArrayList<FormField>();

fields.add(repoField);

fields.add(resourceStorePathField);
fields.add(modifiedSinceDateField);
fields.add(walkingLimitTpsField);

return fields;
}
Expand Down
22 changes: 21 additions & 1 deletion components/nexus-core/src/test/java/org/sonatype/nexus/tasks/ReconcileChecksumsTaskTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import org.sonatype.nexus.proxy.maven.MUtils;
import org.sonatype.nexus.scheduling.NexusScheduler;

import org.joda.time.DateTime;
import org.junit.Test;

import static org.hamcrest.MatcherAssert.assertThat;
Expand Down Expand Up @@ -94,8 +95,27 @@ public void testChecksumsAreReconciled() throws Exception {
assertThat(MUtils.readDigestFromFileItem((StorageFileItem) snapshots.retrieveItem(metadataMD5)),
is("ec91980b1f83604ed72d749eebacea53"));

task.setRepositoryId(snapshots.getId());
task.setResourceStorePath("/org/sonatype/");
task.setModifiedSinceDate(DateTime.now().plusYears(1).toString("yyyy-MM-dd"));

nexusScheduler.submit("testAffectedSubTree", task).get();

item = snapshots.retrieveItem(metadataXML);

sha1sum = item.getRepositoryItemAttributes().get(DigestCalculatingInspector.DIGEST_SHA1_KEY);
md5sum = item.getRepositoryItemAttributes().get(DigestCalculatingInspector.DIGEST_MD5_KEY);

// checksums still need to be reconciled
assertThat(sha1sum, is("18da84408c713e55a3c55dc34a1ccff78086fd46"));
assertThat(md5sum, is("ec91980b1f83604ed72d749eebacea53"));

// associated checksum files still need to be reconciled
assertThat(MUtils.readDigestFromFileItem((StorageFileItem) snapshots.retrieveItem(metadataSHA1)),
is("18da84408c713e55a3c55dc34a1ccff78086fd46"));
assertThat(MUtils.readDigestFromFileItem((StorageFileItem) snapshots.retrieveItem(metadataMD5)),
is("ec91980b1f83604ed72d749eebacea53"));

task.setModifiedSinceDate("2015-01-01");

nexusScheduler.submit("testAffectedSubTree", task).get();

Expand Down

0 comments on commit 63e389d

Please sign in to comment.