Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add support for Amazon S3 as a lock storage #2325

Merged
merged 5 commits into from
Jan 4, 2025
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Fix code format
  • Loading branch information
caiooliveiraeti committed Jan 2, 2025
commit 8fbbc28f2898c81795db0ac9413d35e84b0810ff
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -905,7 +905,7 @@ import net.javacrumbs.shedlock.provider.s3.S3LockProvider;

@Bean
public LockProvider lockProvider(com.amazonaws.services.s3.AmazonS3 amazonS3) {
return new S3LockProvider(amazonS3);
return new S3LockProvider(amazonS3, "BUKET_NAME");
}
```

Expand Down
1 change: 0 additions & 1 deletion providers/s3/shedlock-provider-s3/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -74,5 +74,4 @@
</plugin>
</plugins>
</build>

</project>
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
package net.javacrumbs.shedlock.provider.s3;

import com.amazonaws.services.s3.AmazonS3;
import net.javacrumbs.shedlock.support.StorageBasedLockProvider;

Expand Down Expand Up @@ -28,4 +29,3 @@ public S3LockProvider(AmazonS3 s3Client, String bucketName) {
this(s3Client, bucketName, "shedlock/");
}
}

Original file line number Diff line number Diff line change
@@ -1,18 +1,17 @@
package net.javacrumbs.shedlock.provider.s3;

import com.amazonaws.AmazonServiceException;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.PutObjectResult;
import net.javacrumbs.shedlock.core.ClockProvider;
import net.javacrumbs.shedlock.core.LockConfiguration;
import net.javacrumbs.shedlock.support.AbstractStorageAccessor;

import java.io.ByteArrayInputStream;
import java.time.Instant;
import java.util.Optional;
import java.util.UUID;
import net.javacrumbs.shedlock.core.ClockProvider;
import net.javacrumbs.shedlock.core.LockConfiguration;
import net.javacrumbs.shedlock.support.AbstractStorageAccessor;

/**
* Implementation of StorageAccessor for S3 as a lock storage backend.
Expand Down Expand Up @@ -66,12 +65,12 @@ public boolean insertRecord(LockConfiguration lockConfiguration) {

try {
var lockContent = UUID.randomUUID().toString().getBytes();
ObjectMetadata metadata = createMetadata(lockConfiguration.getLockAtMostUntil(), ClockProvider.now(), getHostname());
ObjectMetadata metadata =
createMetadata(lockConfiguration.getLockAtMostUntil(), ClockProvider.now(), getHostname());
metadata.setContentLength(lockContent.length);

PutObjectRequest request = new PutObjectRequest(
bucketName, objectName(name), new ByteArrayInputStream(lockContent), metadata
);
PutObjectRequest request =
new PutObjectRequest(bucketName, objectName(name), new ByteArrayInputStream(lockContent), metadata);
request.putCustomRequestHeader("If-None-Match", "*");

s3Client.putObject(request);
Expand All @@ -91,12 +90,17 @@ bucketName, objectName(name), new ByteArrayInputStream(lockContent), metadata
public boolean updateRecord(LockConfiguration lockConfiguration) {
Optional<Lock> lock = find(lockConfiguration.getName(), "updateRecord");
if (lock.isEmpty() || lock.get().lockUntil().isAfter(ClockProvider.now())) {
logger.debug("Update skipped. Lock still valid or not found. name: {}, lock: {}", lockConfiguration.getName(), lock);
logger.debug(
"Update skipped. Lock still valid or not found. name: {}, lock: {}",
lockConfiguration.getName(),
lock);
return false;
}

ObjectMetadata newMetadata = createMetadata(lockConfiguration.getLockAtMostUntil(), ClockProvider.now(), getHostname());
return replaceObjectMetadata(lockConfiguration.getName(), newMetadata, lock.get().eTag(), "updateRecord");
ObjectMetadata newMetadata =
createMetadata(lockConfiguration.getLockAtMostUntil(), ClockProvider.now(), getHostname());
return replaceObjectMetadata(
lockConfiguration.getName(), newMetadata, lock.get().eTag(), "updateRecord");
}

@Override
Expand All @@ -116,7 +120,10 @@ public boolean extend(LockConfiguration lockConfiguration) {
if (lock.isEmpty()
|| lock.get().lockUntil().isBefore(ClockProvider.now())
|| !lock.get().lockedBy().equals(getHostname())) {
logger.debug("Extend skipped. Lock invalid or not owned by host. name: {}, lock: {}", lockConfiguration.getName(), lock);
logger.debug(
"Extend skipped. Lock invalid or not owned by host. name: {}, lock: {}",
lockConfiguration.getName(),
lock);
return false;
}

Expand All @@ -125,7 +132,8 @@ public boolean extend(LockConfiguration lockConfiguration) {

private boolean updateUntil(String name, Lock lock, Instant until, String action) {
ObjectMetadata existingMetadata = s3Client.getObjectMetadata(bucketName, objectName(name));
ObjectMetadata newMetadata = createMetadata(until, Instant.parse(existingMetadata.getUserMetaDataOf(LOCKED_AT)), getHostname());
ObjectMetadata newMetadata =
createMetadata(until, Instant.parse(existingMetadata.getUserMetaDataOf(LOCKED_AT)), getHostname());

return replaceObjectMetadata(name, newMetadata, lock.eTag(), action);
}
Expand All @@ -134,14 +142,18 @@ private boolean replaceObjectMetadata(String name, ObjectMetadata newMetadata, S
var lockContent = UUID.randomUUID().toString().getBytes();
newMetadata.setContentLength(lockContent.length);

PutObjectRequest request = new PutObjectRequest(
bucketName, objectName(name), new ByteArrayInputStream(lockContent), newMetadata
);
PutObjectRequest request =
new PutObjectRequest(bucketName, objectName(name), new ByteArrayInputStream(lockContent), newMetadata);
request.putCustomRequestHeader("If-Match", eTag);

try {
PutObjectResult response = s3Client.putObject(request);
logger.debug("Lock {} successfully. name: {}, old e-tag: {}, new e-tag: {}", action, name, eTag, response.getETag());
logger.debug(
"Lock {} successfully. name: {}, old e-tag: {}, new e-tag: {}",
action,
name,
eTag,
response.getETag());
return true;
} catch (AmazonServiceException e) {
if (e.getStatusCode() == 412) {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package net.javacrumbs.shedlock.provider.s3;

import static net.javacrumbs.shedlock.core.ClockProvider.now;

import static org.assertj.core.api.Assertions.assertThat;

import com.amazonaws.auth.AWSStaticCredentialsProvider;
Expand Down Expand Up @@ -30,19 +29,19 @@ public class S3LockProviderIntegrationTest extends AbstractStorageBasedLockProvi

@Container
public static final MyLocalStackS3Container localStackS3 = new MyLocalStackS3Container();

private static AmazonS3 s3Client;
private static final String BUCKET_NAME = "my-bucket";
private static final String OBJECT_PREFIX = "prefix";

@BeforeAll
public static void startLocalStackS3() {
s3Client = AmazonS3ClientBuilder.standard().withEndpointConfiguration(
new AwsClientBuilder.EndpointConfiguration(localStackS3.getEndpoint().toString(), localStackS3.getRegion())
).withCredentials(
new AWSStaticCredentialsProvider(
new BasicAWSCredentials(localStackS3.getAccessKey(), localStackS3.getSecretKey())
)
).build();
s3Client = AmazonS3ClientBuilder.standard()
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
localStackS3.getEndpoint().toString(), localStackS3.getRegion()))
.withCredentials(new AWSStaticCredentialsProvider(
new BasicAWSCredentials(localStackS3.getAccessKey(), localStackS3.getSecretKey())))
.build();
}

@BeforeEach
Expand All @@ -52,7 +51,7 @@ public void before() {

@AfterEach
public void after() {
s3Client.listObjects(BUCKET_NAME).getObjectSummaries().forEach(obj ->{
s3Client.listObjects(BUCKET_NAME).getObjectSummaries().forEach(obj -> {
s3Client.deleteObject(BUCKET_NAME, obj.getKey());
});
}
Expand All @@ -79,7 +78,9 @@ protected void assertLocked(String lockName) {
}

private Lock findLock(String lockName) {
return new S3StorageAccessor(s3Client, BUCKET_NAME, OBJECT_PREFIX).find(lockName, "test").get();
return new S3StorageAccessor(s3Client, BUCKET_NAME, OBJECT_PREFIX)
.find(lockName, "test")
.get();
}

private static class MyLocalStackS3Container extends LocalStackContainer {
Expand All @@ -88,4 +89,3 @@ public MyLocalStackS3Container() {
}
}
}

Loading