Skip to content

Commit

Permalink
Merge pull request #951 from amvanbaren/ratelimit-signature-jobs
Browse files Browse the repository at this point in the history 
use streams to limit memory usage
  • Loading branch information
@amvanbaren
amvanbaren authored Jun 24, 2024
2 parents a53f1c6 + 4b4bfc3 commit f711670
Show file tree
Hide file tree
Showing 7 changed files with 52 additions and 44 deletions.
6 changes: 5 additions & 1 deletion server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import org.eclipse.openvsx.entities.Extension;
import org.eclipse.openvsx.migration.HandlerJobRequest;
import org.eclipse.openvsx.util.NamingUtil;
import org.eclipse.openvsx.util.TimeUtil;
import org.eclipse.openvsx.util.UrlUtil;
import org.jobrunr.scheduling.JobRequestScheduler;
import org.jobrunr.scheduling.cron.Cron;
Expand Down Expand Up @@ -49,6 +50,9 @@ public class VSCodeIdService {
@Value("${ovsx.vscode.upstream.update-on-start:false}")
boolean updateOnStart;

@Value("${ovsx.migrations.delay.seconds:0}")
long delay;

public VSCodeIdService(
RestTemplate vsCodeIdRestTemplate,
UrlConfigService urlConfigService,
Expand All @@ -65,7 +69,7 @@ public void applicationStarted(ApplicationStartedEvent event) {
return;
}
if(updateOnStart) {
scheduler.enqueue(new HandlerJobRequest<>(VSCodeIdDailyUpdateJobRequestHandler.class));
scheduler.schedule(TimeUtil.getCurrentUTC().plusSeconds(delay), new HandlerJobRequest<>(VSCodeIdDailyUpdateJobRequestHandler.class));
}

scheduler.scheduleRecurrently("VSCodeIdDailyUpdate", Cron.daily(3), ZoneId.of("UTC"), new HandlerJobRequest<>(VSCodeIdDailyUpdateJobRequestHandler.class));
Expand Down
2 changes: 1 addition & 1 deletion server/src/main/java/org/eclipse/openvsx/admin/AdminStatisticsJobRequest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,4 +44,4 @@ public void setMonth(int month) {
public Class<? extends JobRequestHandler> getJobRequestHandler() {
return AdminStatisticsJobRequestHandler.class;
}
}
}
2 changes: 1 addition & 1 deletion server/src/main/java/org/eclipse/openvsx/admin/AdminStatisticsJobRequestHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,4 +119,4 @@ public void run(AdminStatisticsJobRequest jobRequest) throws Exception {
statistics.setTopMostDownloadedExtensions(topMostDownloadedExtensions);
service.saveAdminStatistics(statistics);
}
}
}
5 changes: 2 additions & 3 deletions server/src/main/java/org/eclipse/openvsx/migration/GenerateKeyPairJobRequestHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,13 +83,12 @@ private void deleteKeyPairs() {

private void enqueueCreateSignatureJob(ExtensionVersion extVersion) {
var handler = ExtensionVersionSignatureJobRequestHandler.class;
var jobRequest = new MigrationJobRequest<>(handler, extVersion.getId());
scheduler.schedule(TimeUtil.getCurrentUTC().plusSeconds(30), jobRequest);
scheduler.enqueue(new MigrationJobRequest<>(handler, extVersion.getId()));
}

private void enqueueDeleteSignatureJob(FileResource resource) {
if(!resource.getStorageType().equals(STORAGE_DB)) {
scheduler.schedule(TimeUtil.getCurrentUTC().plusSeconds(30), new RemoveFileJobRequest(resource));
scheduler.enqueue(new RemoveFileJobRequest(resource));
}
}
}
34 changes: 14 additions & 20 deletions server/src/main/java/org/eclipse/openvsx/mirror/MirrorExtensionService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,7 @@
import java.time.LocalDate;
import java.util.*;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipFile;

import static org.eclipse.openvsx.entities.FileResource.DOWNLOAD_SIG;
import static org.eclipse.openvsx.entities.FileResource.PUBLIC_KEY;
Expand Down Expand Up @@ -177,10 +176,10 @@ private void mirrorExtensionVersion(ExtensionJson json) throws RuntimeException
if(json.files.containsKey(DOWNLOAD_SIG)) {
try(
var signatureZip = downloadToFile(json.files.get(DOWNLOAD_SIG), "extension_", ".sigzip");
var signature = extractSignature(signatureZip);
var signatureFile = extractSignature(signatureZip);
var publicKeyFile = downloadToFile(json.files.get(PUBLIC_KEY), "public_", ".pem");
) {
var verified = integrityService.verifyExtensionVersion(extensionFile, signature, publicKeyFile);
var verified = integrityService.verifyExtensionVersion(extensionFile, signatureFile, publicKeyFile);
if (!verified) {
throw new RuntimeException("Unverified vsix package");
}
Expand Down Expand Up @@ -220,24 +219,19 @@ private TempFile downloadToFile(String url, String prefix, String suffix) throws

private TempFile extractSignature(TempFile signatureZip) throws RuntimeException, IOException {
var signature = new TempFile("extension_",".signature.sig");
try(var zipInput = new ZipInputStream(Files.newInputStream(signatureZip.getPath()))) {
ZipEntry zipEntry = zipInput.getNextEntry();
while (zipEntry != null) {
if (zipEntry.getName().endsWith(".signature.sig")) {
try (var out = Files.newOutputStream(signature.getPath())) {
int len;
byte[] buffer = new byte[1024];
while ((len = zipInput.read(buffer)) > 0) {
out.write(buffer, 0, len);
}
return signature;
}
}

zipEntry = zipInput.getNextEntry();
try (var zipFile = new ZipFile(signatureZip.getPath().toFile())) {
var sigEntry = zipFile.getEntry(".signature.sig");
if(sigEntry == null) {
throw new RuntimeException("No extension signature found");
}
try (
var sigInput = zipFile.getInputStream(sigEntry);
var sigOut = Files.newOutputStream(signature.getPath())
) {
sigInput.transferTo(sigOut);
}
}

throw new RuntimeException("No extension signature found");
return signature;
}
}
42 changes: 26 additions & 16 deletions server/src/main/java/org/eclipse/openvsx/publish/ExtensionVersionIntegrityService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.micrometer.observation.ObservationRegistry;
import jakarta.persistence.EntityManager;
import jakarta.transaction.Transactional;
import org.apache.commons.codec.binary.Base64;
Expand All @@ -26,7 +25,6 @@
import org.eclipse.openvsx.entities.ExtensionVersion;
import org.eclipse.openvsx.entities.FileResource;
import org.eclipse.openvsx.entities.SignatureKeyPair;
import org.eclipse.openvsx.util.ArchiveUtil;
import org.eclipse.openvsx.util.ErrorResultException;
import org.eclipse.openvsx.util.NamingUtil;
import org.eclipse.openvsx.util.TempFile;
Expand All @@ -39,6 +37,7 @@

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
Expand Down Expand Up @@ -115,17 +114,11 @@ public FileResource generateSignature(FileResource download, TempFile extensionF
resource.setExtension(download.getExtension());
resource.setName(NamingUtil.toFileFormat(download.getExtension(), ".sigzip"));
resource.setType(FileResource.DOWNLOAD_SIG);

var privateKeyParameters = new Ed25519PrivateKeyParameters(keyPair.getPrivateKey(), 0);
try (var out = new ByteArrayOutputStream()) {
try (var zip = new ZipOutputStream(out)) {
var signer = new Ed25519Signer();
signer.init(true, privateKeyParameters);
var fileBytes = Files.readAllBytes(extensionFile.getPath());
signer.update(fileBytes, 0, fileBytes.length);
var sigEntry = new ZipEntry(".signature.sig");
zip.putNextEntry(sigEntry);
zip.write(signer.generateSignature());
zip.write(generateSignature(extensionFile, keyPair));
zip.closeEntry();

var manifestEntry = new ZipEntry(".signature.manifest");
Expand All @@ -148,6 +141,21 @@ public FileResource generateSignature(FileResource download, TempFile extensionF
return resource;
}

private byte[] generateSignature(TempFile extensionFile, SignatureKeyPair keyPair) throws IOException {
var privateKeyParameters = new Ed25519PrivateKeyParameters(keyPair.getPrivateKey(), 0);
var signer = new Ed25519Signer();
signer.init(true, privateKeyParameters);
try (var in = Files.newInputStream(extensionFile.getPath())) {
int len;
var buffer = new byte[1024];
while ((len = in.read(buffer)) > 0) {
signer.update(buffer, 0, len);
}
}

return signer.generateSignature();
}

private byte[] generateSignatureManifest(TempFile extensionFile) throws IOException {
var base64 = new Base64();
var mapper = new ObjectMapper();
Expand All @@ -156,27 +164,29 @@ private byte[] generateSignatureManifest(TempFile extensionFile) throws IOExcept
zip.stream()
.filter(entry -> !entry.isDirectory())
.forEach(entry -> {
try {
var manifestEntry = generateManifestEntry(zip.getInputStream(entry).readAllBytes(), mapper, base64);
try (var entryStream = zip.getInputStream(entry)) {
var manifestEntry = generateManifestEntry(entryStream, entry.getSize(), mapper, base64);
manifestEntries.set(new String(base64.encode(entry.getName().getBytes(StandardCharsets.UTF_8))), manifestEntry);
} catch (IOException e) {
logger.warn("Failed to add entry to manifest", e);
throw new RuntimeException(e);
}
});
}

var manifest = mapper.createObjectNode();
manifest.set("package", generateManifestEntry(Files.readAllBytes(extensionFile.getPath()), mapper, base64));
try (var extensionStream = Files.newInputStream(extensionFile.getPath())) {
manifest.set("package", generateManifestEntry(extensionStream, Files.size(extensionFile.getPath()), mapper, base64));
}
manifest.set("entries", manifestEntries);
return mapper.writeValueAsBytes(manifest);
}

private JsonNode generateManifestEntry(byte[] content, ObjectMapper mapper, Base64 base64) {
private JsonNode generateManifestEntry(InputStream stream, long size, ObjectMapper mapper, Base64 base64) throws IOException {
var manifestEntry = mapper.createObjectNode();
manifestEntry.put("size", content.length);
manifestEntry.put("size", size);

var manifestEntryDigests = mapper.createObjectNode();
var sha256 = new String(base64.encode(DigestUtils.sha256(content)));
var sha256 = new String(base64.encode(DigestUtils.sha256(stream)));
manifestEntryDigests.put("sha256", sha256);
manifestEntry.set("digests", manifestEntryDigests);
return manifestEntry;
Expand Down
5 changes: 3 additions & 2 deletions server/src/test/java/org/eclipse/openvsx/publish/ExtensionVersionIntegrityServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,10 @@ public void testGenerateSignature() throws IOException {
var sigzipContent = new byte[0];
try (
var stream = getClass().getResource("ms-python.python-2024.7.11511013.vsix").openStream();
var extensionFile = new TempFile("ms-python", ".vsix")
var extensionFile = new TempFile("ms-python", ".vsix");
var out = Files.newOutputStream(extensionFile.getPath())
) {
Files.write(extensionFile.getPath(), stream.readAllBytes());
stream.transferTo(out);
var signature = integrityService.generateSignature(download, extensionFile, keyPair);
sigzipContent = signature.getContent();
}
Expand Down

0 comments on commit f711670

Please sign in to comment.