Skip to content

Commit

Permalink
Merge branch 'master' into sync-noir
Browse files Browse the repository at this point in the history 
* master:
  feat: Dynamic assertion payloads v2 (#5949)
  doc: Fix grammar error on avm spec
  refactor: `create_fixed_base_constraint` cleanup (#6047)
  feat: avm mem trace validation (#6025)
  git subrepo push --branch=master noir-projects/aztec-nr
  git_subrepo.sh: Fix parent in .gitrepo file. [skip ci]
  chore: replace relative paths to noir-protocol-circuits
  git subrepo push --branch=master barretenberg
  fix(avm): Do not scale CALLDATACOPY base cost with size (#5879)
  fix: hotfix stopped instance terminate (#6037)
  chore: clean up stopped instances (#6030)
  • Loading branch information
@TomAFrench
TomAFrench committed Apr 26, 2024
2 parents d40c0ee + 405bdf6 commit 1e5ba2b
Show file tree
Hide file tree
Showing 98 changed files with 2,440 additions and 968 deletions.
126 changes: 29 additions & 97 deletions .github/spot-runner-action/dist/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,15 +200,6 @@ class Ec2Instance {
}
});
}
// async runInstances(params: RunInstancesRequest) {
// const client = await this.getEc2Client();
// try {
// return (await client.runInstances(params).promise()).Instances;
// } catch (error) {
// core.error(`Failed to create instance(s)`);
// throw error;
// }
// }
getSubnetAzId() {
var _a;
return __awaiter(this, void 0, void 0, function* () {
Expand Down Expand Up @@ -329,82 +320,12 @@ class Ec2Instance {
DefaultTargetCapacityType: useOnDemand ? "on-demand" : "spot",
},
};
// const config: SpotFleetRequestConfigData = {
// IamFleetRole:
// "arn:aws:iam::278380418400:role/aws-ec2-spot-fleet-tagging-role",
// TargetCapacity: 1,
// // We always ask for 1 instance, but might ask for 100% on demand or spot
// OnDemandTargetCapacity: useOnDemand ? 1 : 0,
// TerminateInstancesWithExpiration: true,
// Type: "request",
// LaunchSpecifications:
// };
// const params: RequestSpotFleetRequest = {
// SpotFleetRequestConfig: config,
// };
const client = yield this.getEc2Client();
const fleet = yield client.createFleet(createFleetRequest).promise();
const instances = ((fleet === null || fleet === void 0 ? void 0 : fleet.Instances) || [])[0] || {};
return (instances.InstanceIds || [])[0];
});
}
// async getOnDemandInstanceConfiguration(
// ec2SpotInstanceStrategy: string
// ): Promise<RunInstancesRequest> {
// const userData = new UserData(this.config);
// const params: RunInstancesRequest = {
// ImageId: this.config.ec2AmiId,
// InstanceInitiatedShutdownBehavior: "terminate",
// InstanceMarketOptions: {},
// InstanceType: "",
// MaxCount: 1,
// MinCount: 1,
// SecurityGroupIds: [this.config.ec2SecurityGroupId],
// SubnetId: this.config.ec2SubnetId,
// KeyName: this.config.ec2KeyName,
// Placement: {
// AvailabilityZone: await this.getSubnetAz(),
// },
// TagSpecifications: [
// {
// ResourceType: "instance",
// Tags: this.tags,
// },
// ],
// // <aztec>parity with build-system
// BlockDeviceMappings: [
// {
// DeviceName: "/dev/sda1",
// Ebs: {
// VolumeSize: 32,
// },
// },
// ],
// // parity with build-system</aztec>
// UserData: await userData.getUserData(),
// };
// switch (ec2SpotInstanceStrategy.toLowerCase()) {
// case "besteffort":
// case "spotonly": {
// params.InstanceMarketOptions = {
// MarketType: "spot",
// SpotOptions: {
// InstanceInterruptionBehavior: "terminate",
// SpotInstanceType: "one-time",
// },
// };
// break;
// }
// case "none": {
// params.InstanceMarketOptions = {};
// break;
// }
// default: {
// throw new TypeError("Invalid value for ec2_spot_instance_strategy");
// }
// }
// return params;
// }
getInstanceStatus(instanceId) {
return __awaiter(this, void 0, void 0, function* () {
const client = yield this.getEc2Client();
Expand All @@ -420,7 +341,7 @@ class Ec2Instance {
}
});
}
getInstancesForTags() {
getInstancesForTags(instanceStatus) {
return __awaiter(this, void 0, void 0, function* () {
const client = yield this.getEc2Client();
const filters = [
Expand All @@ -438,6 +359,10 @@ class Ec2Instance {
for (const reservation of (yield client.describeInstances(params).promise()).Reservations || []) {
instances = instances.concat(reservation.Instances || []);
}
if (instanceStatus) {
// Filter instances that are stopped
instances = instances.filter((instance) => { var _a; return ((_a = instance === null || instance === void 0 ? void 0 : instance.State) === null || _a === void 0 ? void 0 : _a.Name) === instanceStatus; });
}
return instances;
}
catch (error) {
Expand Down Expand Up @@ -716,9 +641,8 @@ function pollSpotStatus(config, ec2Client, ghClient) {
return __awaiter(this, void 0, void 0, function* () {
// 12 iters x 10000 ms = 2 minutes
for (let iter = 0; iter < 12; iter++) {
const instances = yield ec2Client.getInstancesForTags();
const hasInstance = instances.filter((i) => { var _a; return ((_a = i.State) === null || _a === void 0 ? void 0 : _a.Name) === "running"; }).length > 0;
if (!hasInstance) {
const instances = yield ec2Client.getInstancesForTags("running");
if (instances.length <= 0) {
// we need to start an instance
return "none";
}
Expand All @@ -742,14 +666,18 @@ function start() {
return __awaiter(this, void 0, void 0, function* () {
const config = new config_1.ActionConfig();
if (config.subaction === "stop") {
yield stop();
yield terminate();
return;
}
else if (config.subaction === "restart") {
yield stop();
yield terminate();
// then we make a fresh instance
}
else if (config.subaction !== "start") {
else if (config.subaction === "start") {
// We need to terminate
yield terminate("stopped", false);
}
else {
throw new Error("Unexpected subaction: " + config.subaction);
}
// subaction is 'start' or 'restart'estart'
Expand All @@ -765,7 +693,7 @@ function start() {
if (config.subaction === "restart") {
throw new Error("Taking down spot we just started. This seems wrong, erroring out.");
}
yield stop();
yield terminate();
}
var ec2SpotStrategies;
switch (config.ec2SpotInstanceStrategy) {
Expand Down Expand Up @@ -831,22 +759,26 @@ function start() {
}
});
}
function stop() {
function terminate(instanceStatus, cleanupRunners = true) {
return __awaiter(this, void 0, void 0, function* () {
try {
core.info("Starting instance cleanup");
const config = new config_1.ActionConfig();
const ec2Client = new ec2_1.Ec2Instance(config);
const ghClient = new github_1.GithubClient(config);
const instances = yield ec2Client.getInstancesForTags();
const instances = yield ec2Client.getInstancesForTags(instanceStatus);
yield ec2Client.terminateInstances(instances.map((i) => i.InstanceId));
core.info("Clearing previously installed runners");
const result = yield ghClient.removeRunnersWithLabels([config.githubJobId]);
if (result) {
core.info("Finished runner cleanup");
}
else {
throw Error("Failed to cleanup runners. Continuing, but failure expected!");
if (cleanupRunners) {
core.info("Clearing previously installed runners");
const result = yield ghClient.removeRunnersWithLabels([
config.githubJobId,
]);
if (result) {
core.info("Finished runner cleanup");
}
else {
throw Error("Failed to cleanup runners. Continuing, but failure expected!");
}
}
}
catch (error) {
Expand All @@ -860,7 +792,7 @@ function stop() {
start();
}
catch (error) {
stop();
terminate();
(0, utils_1.assertIsError)(error);
core.error(error);
core.setFailed(error.message);
Expand Down
93 changes: 7 additions & 86 deletions .github/spot-runner-action/src/ec2.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,17 +117,6 @@ export class Ec2Instance {
}
}

// async runInstances(params: RunInstancesRequest) {
// const client = await this.getEc2Client();

// try {
// return (await client.runInstances(params).promise()).Instances;
// } catch (error) {
// core.error(`Failed to create instance(s)`);
// throw error;
// }
// }

async getSubnetAzId() {
const client = await this.getEc2Client();
try {
Expand Down Expand Up @@ -252,86 +241,12 @@ export class Ec2Instance {
DefaultTargetCapacityType: useOnDemand ? "on-demand" : "spot",
},
};
// const config: SpotFleetRequestConfigData = {
// IamFleetRole:
// "arn:aws:iam::278380418400:role/aws-ec2-spot-fleet-tagging-role",
// TargetCapacity: 1,
// // We always ask for 1 instance, but might ask for 100% on demand or spot
// OnDemandTargetCapacity: useOnDemand ? 1 : 0,
// TerminateInstancesWithExpiration: true,
// Type: "request",
// LaunchSpecifications:
// };
// const params: RequestSpotFleetRequest = {
// SpotFleetRequestConfig: config,
// };
const client = await this.getEc2Client();
const fleet = await client.createFleet(createFleetRequest).promise();
const instances: CreateFleetInstance = (fleet?.Instances || [])[0] || {};
return (instances.InstanceIds || [])[0];
}

// async getOnDemandInstanceConfiguration(
// ec2SpotInstanceStrategy: string
// ): Promise<RunInstancesRequest> {
// const userData = new UserData(this.config);

// const params: RunInstancesRequest = {
// ImageId: this.config.ec2AmiId,
// InstanceInitiatedShutdownBehavior: "terminate",
// InstanceMarketOptions: {},
// InstanceType: "",
// MaxCount: 1,
// MinCount: 1,
// SecurityGroupIds: [this.config.ec2SecurityGroupId],
// SubnetId: this.config.ec2SubnetId,
// KeyName: this.config.ec2KeyName,
// Placement: {
// AvailabilityZone: await this.getSubnetAz(),
// },
// TagSpecifications: [
// {
// ResourceType: "instance",
// Tags: this.tags,
// },
// ],
// // <aztec>parity with build-system
// BlockDeviceMappings: [
// {
// DeviceName: "/dev/sda1",
// Ebs: {
// VolumeSize: 32,
// },
// },
// ],
// // parity with build-system</aztec>
// UserData: await userData.getUserData(),
// };

// switch (ec2SpotInstanceStrategy.toLowerCase()) {
// case "besteffort":
// case "spotonly": {
// params.InstanceMarketOptions = {
// MarketType: "spot",
// SpotOptions: {
// InstanceInterruptionBehavior: "terminate",
// SpotInstanceType: "one-time",
// },
// };
// break;
// }
// case "none": {
// params.InstanceMarketOptions = {};
// break;
// }
// default: {
// throw new TypeError("Invalid value for ec2_spot_instance_strategy");
// }
// }

// return params;
// }

async getInstanceStatus(instanceId: string) {
const client = await this.getEc2Client();
try {
Expand All @@ -347,7 +262,7 @@ export class Ec2Instance {
}
}

async getInstancesForTags(): Promise<AWS.EC2.Instance[]> {
async getInstancesForTags(instanceStatus?: string): Promise<AWS.EC2.Instance[]> {
const client = await this.getEc2Client();
const filters: FilterInterface[] = [
{
Expand All @@ -367,6 +282,12 @@ export class Ec2Instance {
).Reservations || []) {
instances = instances.concat(reservation.Instances || []);
}
if (instanceStatus) {
// Filter instances that are stopped
instances = instances.filter(
(instance) => instance?.State?.Name === instanceStatus
);
}
return instances;
} catch (error) {
core.error(
Expand Down
Loading

0 comments on commit 1e5ba2b

Please sign in to comment.