Skip to content

Commit

Permalink
feat(webserver): Marketplace proxy
Browse files Browse the repository at this point in the history
  • Loading branch information
brian-mulier-p committed Nov 28, 2023
1 parent e9d6634 commit 073af36
Show file tree
Hide file tree
Showing 13 changed files with 442 additions and 23 deletions.
5 changes: 5 additions & 0 deletions cli/src/main/resources/application.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,11 @@ micronaut:
connect-timeout: 180s
read-timeout: 180s

proxy:
read-idle-timeout: 180s
connect-timeout: 180s
read-timeout: 180s

# By default, Micronaut uses a scheduled executor with 2*nbProc for @Scheduled which is a lot as we didn't use much scheduling tasks.
# Using core-pool-size to set the minimum nb threads to keep when idle instead.
executors:
Expand Down
61 changes: 39 additions & 22 deletions ui/public/init.js
Original file line number Diff line number Diff line change
Expand Up @@ -14,19 +14,27 @@ require.config({
});

// publisherId/extensionId format
const versionByExtensionIdToFetch = {
const extensionsToFetch = {
// to handle dark theme
"PROxZIMA/sweetdracula": "1.0.9",
// to apply Kestra's flow validation schema
"kestra-io/kestra": "0.1.7",
// for Python autocompletion along with Pylance that is needed for it to work
// "ms-python/python": "2023.20.0"
};

const extensionsToFetch = Object.entries(versionByExtensionIdToFetch).map(([extensionId, version]) => ({
scheme: "https",
authority: "openvsxorg.blob.core.windows.net",
path: `/resources/${extensionId}/${version}/extension`
let url;
try {
url = new URL(KESTRA_API_URL);
} catch (e) {
// KESTRA_API_URL is a relative path
url = {
...window.location,
pathname: KESTRA_API_URL
};
}
const extensionUrls = Object.entries(extensionsToFetch).map(([extensionId, version]) => ({
scheme: url.protocol.replace(":", ""),
authority: url.host,
path: `${url.pathname}/editor/marketplace/resource/${extensionId}/${version}/extension`
}));

// used to configure VSCode startup
Expand Down Expand Up @@ -55,16 +63,34 @@ const bottomBarTabs = [
{"id":"refactorPreview", "pinned": false,"visible": false}
];

const apiUrl = url.origin + url.pathname;
window.product = {
productConfiguration: {
nameShort: "Kestra VSCode",
nameLong: "Kestra VSCode",
// configure the open sx marketplace
"extensionsGallery": {
"serviceUrl": "https://open-vsx.org/vscode/gallery",
"itemUrl": "https://open-vsx.org/vscode/item",
"resourceUrlTemplate": "https://openvsxorg.blob.core.windows.net/resources/{publisher}/{name}/{version}/{path}"
// configure VSCode Marketplace
extensionsGallery: {
nlsBaseUrl: `${apiUrl}/editor/marketplace/nls`,
serviceUrl: `${apiUrl}/editor/marketplace/service`,
searchUrl: `${apiUrl}/editor/marketplace/search`,
servicePPEUrl: `${apiUrl}/editor/marketplace/serviceppe`,
cacheUrl: `${apiUrl}/editor/marketplace/cache`,
itemUrl: `${apiUrl}/editor/marketplace/item`,
publisherUrl: `${apiUrl}/editor/marketplace/publisher`,
resourceUrlTemplate: `${apiUrl}/editor/marketplace/resource/{publisher}/{name}/{version}/{path}`,
controlUrl: `${apiUrl}/editor/marketplace/control`
},
extensionEnabledApiProposals: {
"ms-python.python": [
"contribEditorContentMenu",
"quickPickSortByLabel",
"testObserver",
"quickPickItemTooltip",
"saveEditor",
"terminalDataWriteEvent",
"terminalExecuteCommandEvent"
]
}
},
// scope the VSCode instance to Kestra File System Provider (defined in Kestra VSCode extension)
folderUri: {
Expand All @@ -85,16 +111,7 @@ window.product = {
authority: window.location.host,
path: KESTRA_UI_PATH + "vscode/extensions/yaml/extension"
},
// {
// scheme: window.location.protocol.replace(":", ""),
// authority: window.location.host,
// path: KESTRA_UI_PATH + "vscode/extensions/pylance/extension"
// },
...extensionsToFetch
],
"linkProtectionTrustedDomains": [
"https://open-vsx.org",
"https://openvsxorg.blob.core.windows.net"
...extensionUrls
],
configurationDefaults: {
"files.autoSave": "off",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
package io.kestra.webserver.controllers;

import io.kestra.webserver.controllers.domain.MarketplaceRequestType;
import io.kestra.webserver.services.MarketplaceRequestMapper;
import io.micronaut.core.async.publisher.Publishers;
import io.micronaut.http.*;
import io.micronaut.http.annotation.*;
import io.micronaut.http.client.HttpClient;
import io.micronaut.http.client.annotation.Client;
import io.micronaut.http.server.util.HttpHostResolver;
import io.micronaut.scheduling.TaskExecutors;
import io.micronaut.scheduling.annotation.ExecuteOn;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import jakarta.inject.Inject;
import org.reactivestreams.Publisher;

import javax.annotation.Nullable;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Optional;

@Controller("/api/v1/editor")
public class EditorController {
@Inject
@Client("remote-api")
private HttpClient httpClient;

@Inject
private HttpHostResolver httpHostResolver;

@Inject
private MarketplaceRequestMapper marketplaceRequestMapper;

@ExecuteOn(TaskExecutors.IO)
@Get(uri = "/marketplace/{type}{/path:/.*}", produces = MediaType.APPLICATION_JSON)
@Operation(tags = {"Marketplace"}, summary = "Marketplace extensions operations")
public HttpResponse<String> marketplaceGet(
@Parameter(description = "Type of request") @PathVariable MarketplaceRequestType type,
@Parameter(description = "Additional path") @PathVariable @Nullable String path
) {
// proxied
return null;
}

@ExecuteOn(TaskExecutors.IO)
@Post(uri = "/marketplace/{type}{/path:/.*}", consumes = MediaType.APPLICATION_JSON, produces = MediaType.APPLICATION_JSON)
@Operation(tags = {"Marketplace"}, summary = "Marketplace extensions operations")
public HttpResponse<String> marketplacePost(
@Parameter(description = "Type of request") @PathVariable MarketplaceRequestType type,
@Parameter(description = "Additional path") @PathVariable @Nullable String path
) throws URISyntaxException {
// proxied
return null;
}

@ExecuteOn(TaskExecutors.IO)
@Get(uri = "/marketplace/resource/{publisher}/{extension}/{version}{/path:/.*}")
@Operation(tags = {"Marketplace"}, summary = "Marketplace extensions resources operations")
public Publisher<HttpResponse<String>> marketplaceResource(
@Parameter(description = "Publisher id") @PathVariable String publisher,
@Parameter(description = "Extension name") @PathVariable String extension,
@Parameter(description = "Extension version") @PathVariable String version,
@Parameter(description = "Path of the resource") @PathVariable String path,
HttpRequest<?> httpRequest
) {
String localhost = httpHostResolver.resolve(httpRequest);
String resourceBaseUrl = marketplaceRequestMapper.resourceBaseUrl(publisher);

return Publishers.map(
httpClient.exchange(
httpRequest.mutate()
.uri(URI.create(resourceBaseUrl + "/" + publisher + "/" + extension + "/" + version + path))
.headers(headers -> headers.set("Host", resourceBaseUrl.replaceFirst("https?://([^/]*).*", "$1").toLowerCase())),
String.class
), response -> {
String body = response.body();
if (body == null) {
return response;
}

MutableHttpResponse<String> newResponse = HttpResponse.ok(
path.equals("/extension")
? body.replace(resourceBaseUrl, localhost + "/api/v1/editor/marketplace/resource")
: body
);
return Optional.ofNullable(response.header("Content-Type"))
.map(contentType -> newResponse.header("Content-Type", contentType))
.orElse(newResponse);
}
);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package io.kestra.webserver.controllers.domain;

import com.fasterxml.jackson.annotation.JsonCreator;
import lombok.Getter;

public enum MarketplaceRequestType {
NLS("https://vscode-unpkg.net/_lp"),
SERVICE("https://marketplace.visualstudio.com/_apis/public/gallery"),
SEARCH("https://marketplace.visualstudio.com/_apis/public/gallery/searchrelevancy/extensionquery"),
SERVICEPPE("https://marketplace.vsallin.net/_apis/public/gallery"),
CACHE("https://vscode.blob.core.windows.net/gallery/index"),
ITEM("https://marketplace.visualstudio.com/items"),
PUBLISHER("https://marketplace.visualstudio.com/publishers"),
CONTROL("https://az764295.vo.msecnd.net/extensions/marketplace.json");

@Getter
private final String url;

MarketplaceRequestType(String url) {
this.url = url;
}

@JsonCreator
public static MarketplaceRequestType fromString(String key) {
return key == null
? null
: MarketplaceRequestType.valueOf(key.toUpperCase());
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
package io.kestra.webserver.filter;

import io.kestra.webserver.controllers.domain.MarketplaceRequestType;
import io.kestra.webserver.services.MarketplaceRequestMapper;
import io.micronaut.core.async.publisher.Publishers;
import io.micronaut.http.HttpAttributes;
import io.micronaut.http.HttpRequest;
import io.micronaut.http.MutableHttpRequest;
import io.micronaut.http.MutableHttpResponse;
import io.micronaut.http.annotation.Filter;
import io.micronaut.http.client.ProxyHttpClient;
import io.micronaut.http.client.annotation.Client;
import io.micronaut.http.filter.*;
import io.micronaut.web.router.RouteMatch;
import jakarta.inject.Inject;
import org.reactivestreams.Publisher;

import java.net.URI;
import java.util.Collections;
import java.util.Map;
import java.util.Optional;

@Filter("/api/v1/editor/marketplace/**")
public class MarketplaceFilter implements HttpServerFilter {
@Inject
@Client("proxy")
private ProxyHttpClient httpClient;

@Inject
private MarketplaceRequestMapper marketplaceRequestMapper;

@Override
public int getOrder() {
return ServerFilterPhase.RENDERING.order();
}

@Override
public Publisher<MutableHttpResponse<?>> doFilter(HttpRequest<?> request, ServerFilterChain chain) {
MutableHttpRequest<?> httpRequest = request.mutate();

httpRequest.headers(headers -> {
if (Optional.ofNullable(headers.get("Origin")).map(origin -> !origin.contains("localhost")).orElse(true)) {
headers.set("Origin", "http://localhost:8080");
}
headers.remove("Cookie");
headers.remove("Accept-Encoding");
});

Map<String, Object> matchValues = request.getAttribute(HttpAttributes.ROUTE_MATCH, RouteMatch.class)
.map(RouteMatch::getVariableValues)
.orElse(Collections.emptyMap());
MarketplaceRequestType type = Optional.ofNullable(matchValues.get("type"))
.map(String.class::cast)
.map(MarketplaceRequestType::fromString)
.orElse(null);

String path = Optional.ofNullable(matchValues.get("path")).map(Object::toString).orElse("");

Publisher<MutableHttpResponse<?>> publisher;
if (type == null) {
publisher = chain.proceed(httpRequest);
} else {
httpRequest.uri(URI.create(marketplaceRequestMapper.url(type) + path));

publisher = httpClient.proxy(httpRequest);
}

return Publishers.map(
publisher,
mutableHttpResponse -> mutableHttpResponse.headers(headers -> headers.remove("Access-Control-Allow-Origin"))
);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package io.kestra.webserver.services;

import jakarta.inject.Singleton;
import io.kestra.webserver.controllers.domain.MarketplaceRequestType;

@Singleton
// This mapper is almost a no-op but is necessary to ease testing MarketplaceFilter
public class MarketplaceRequestMapper {
public String url(MarketplaceRequestType type) {
return type.getUrl();
}

public String resourceBaseUrl(String publisher) {
return "https://" + publisher + ".vscode-unpkg.net";
}
}
Loading

0 comments on commit 073af36

Please sign in to comment.