Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor #207

Closed
wants to merge 13 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ jobs:
strategy:
fail-fast: false # don't cancel all jobs if some of them failed
matrix:
node: ["10", "12"]
node: ["16"]
steps:
- uses: actions/checkout@v2
- name: Setup node
Expand Down
9 changes: 5 additions & 4 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,8 @@
"homepage": "https://github.com/dappnode/DAppNodeSDK#readme",
"dependencies": {
"@octokit/rest": "^18.0.12",
"ajv": "^6.12.2",
"ajv-errors": "^1.0.1",
"ajv": "^8.11.0",
"ajv-errors": "^3.0.0",
"async-retry": "^1.2.3",
"chalk": "^2.4.2",
"cli-progress": "^3.8.2",
Expand All @@ -63,7 +63,8 @@
"yargs": "^13.2.4"
},
"devDependencies": {
"@types/ajv-errors": "^1.0.2",
"@types/ajv": "^1.0.0",
"@types/ajv-errors": "^2.0.1",
"@types/async-retry": "^1.4.2",
"@types/chai": "^4.2.11",
"@types/cli-progress": "^3.7.0",
Expand All @@ -87,7 +88,7 @@
"eslint": "^7.3.1",
"mocha": "^6.2.2",
"ts-node": "^8.10.2",
"typescript": "^3.9.5"
"typescript": "^4.6.3"
},
"engines": {
"node": ">=8.0.0"
Expand Down
3 changes: 2 additions & 1 deletion src/commands/from_github.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { CommandModule } from "yargs";
import chalk from "chalk";
import inquirer from "inquirer";
import moment from "moment";
import { CliGlobalOptions, defaultArch, Manifest } from "../types";
import { CliGlobalOptions, defaultArch } from "../types";
import {
contentHashFile,
getImagePath,
Expand All @@ -17,6 +17,7 @@ import { getInstallDnpLink } from "../utils/getLinks";
import { githubGetReleases, GithubRelease } from "../utils/githubGetReleases";
import { ipfsAddDirFromUrls } from "../releaseUploader/ipfsNode/addDirFromUrls";
import { verifyIpfsConnection } from "../releaseUploader/ipfsNode/verifyConnection";
import { Manifest } from "../releaseFiles/manifest/types";

interface CliCommandOptions extends CliGlobalOptions {
repoSlug: string;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
import { fetchPinsGroupedByBranch } from "../../../pinStrategy";
import { cliArgsToReleaseUploaderProvider } from "../../../releaseUploader";
import { readManifest } from "../../../utils/manifest";
import { Github } from "../../../providers/github/Github";
import { PinataPinManager } from "../../../providers/pinata/pinManager";
import { readReleaseFile } from "../../../releaseFiles/readReleaseFile";
import { ReleaseFileType } from "../../../releaseFiles/types";

/**
* Removes all pins associated with a branch that no longer exists
Expand All @@ -13,7 +14,7 @@ export async function cleanPinsFromDeletedBranches({
dir: string;
}): Promise<void> {
// Read manifest from disk to get package name
const { manifest } = readManifest({ dir });
const manifest = readReleaseFile(ReleaseFileType.manifest, { dir });

// Connect to Github Octokit REST API to know existing branches
const github = Github.fromLocal(dir);
Expand All @@ -28,7 +29,10 @@ export async function cleanPinsFromDeletedBranches({
throw Error("Must use pinata for deletePins");
const pinata = new PinataPinManager(releaseUploaderProvider);

const pinsGroupedByBranch = await fetchPinsGroupedByBranch(pinata, manifest);
const pinsGroupedByBranch = await fetchPinsGroupedByBranch(
pinata,
manifest.data
);
for (const { branch, pins } of pinsGroupedByBranch) {
if (branches.find(b => b.name === branch)) continue;

Expand Down
35 changes: 24 additions & 11 deletions src/commands/githubActions/bumpUpstream/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,14 @@ import { branchNameRoot, defaultDir } from "../../../params";
import { Github } from "../../../providers/github/Github";
import { getPrBody, getUpstreamVersionTag, VersionToUpdate } from "./format";
import { shell } from "../../../utils/shell";
import { readManifest, writeManifest } from "../../../utils/manifest";
import { readCompose, writeCompose } from "../../../utils/compose";
import { parseCsv } from "../../../utils/csv";
import { getLocalBranchExists, getGitHead } from "../../../utils/git";
import { arrIsUnique } from "../../../utils/array";
import { buildAndComment } from "../build";
import { closeOldPrs } from "./closeOldPrs";
import { writeReleaseFile } from "../../../releaseFiles/writeReleaseFile";
import { readReleaseFile } from "../../../releaseFiles/readReleaseFile";
import { ReleaseFileType } from "../../../releaseFiles/types";

// This action should be run periodically

Expand All @@ -28,11 +29,13 @@ export const gaBumpUpstream: CommandModule<
export async function gaBumpUpstreamHandler({
dir = defaultDir
}: CliGlobalOptions): Promise<void> {
const { manifest, format } = readManifest({ dir });
const compose = readCompose({ dir });
const manifest = readReleaseFile(ReleaseFileType.manifest, { dir });
const compose = readReleaseFile(ReleaseFileType.compose, { dir });

const upstreamRepos = parseCsv(manifest.upstreamRepo);
const upstreamArgs = parseCsv(manifest.upstreamArg || "UPSTREAM_VERSION");
const upstreamRepos = parseCsv(manifest.data.upstreamRepo);
const upstreamArgs = parseCsv(
manifest.data.upstreamArg || "UPSTREAM_VERSION"
);

const githubActor = process.env.GITHUB_ACTOR || "bot";
const userName = githubActor;
Expand Down Expand Up @@ -109,7 +112,7 @@ Compose - ${JSON.stringify(compose, null, 2)}
// index by repoSlug, must be unique
const versionsToUpdateMap = new Map<string, VersionToUpdate>();

for (const [serviceName, service] of Object.entries(compose.services))
for (const [serviceName, service] of Object.entries(compose.data.services))
if (typeof service.build === "object" && service.build.args)
for (const [argName, argValue] of Object.entries(service.build.args)) {
const upstreamRepoVersion = upstreamRepoVersions.get(argName);
Expand All @@ -120,7 +123,7 @@ Compose - ${JSON.stringify(compose, null, 2)}
if (currentVersion === newVersion) continue;

// Update current version
compose.services[serviceName].build = {
compose.data.services[serviceName].build = {
...service.build,
args: {
...service.build.args,
Expand All @@ -141,9 +144,19 @@ Compose - ${JSON.stringify(compose, null, 2)}
}

const versionsToUpdate = Array.from(versionsToUpdateMap.values());
manifest.upstreamVersion = getUpstreamVersionTag(versionsToUpdate);
writeManifest(manifest, format, { dir });
writeCompose(compose, { dir });
manifest.data.upstreamVersion = getUpstreamVersionTag(versionsToUpdate);
writeReleaseFile(
{ type: ReleaseFileType.manifest, data: manifest.data },
manifest.format,
{
dir
}
);
writeReleaseFile(
{ type: ReleaseFileType.compose, data: compose.data },
compose.format,
{ dir }
);

const commitMsg = `bump ${versionsToUpdate
.map(({ repoSlug, newVersion }) => `${repoSlug} to ${newVersion}`)
Expand Down
38 changes: 30 additions & 8 deletions src/commands/init.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@ import chalk from "chalk";
import { CommandModule } from "yargs";
import semver from "semver";
import inquirer from "inquirer";
import { writeManifest, getManifestPath } from "../utils/manifest";
import { writeCompose, getComposePath } from "../utils/compose";
import defaultAvatar from "../assets/defaultAvatar";
import { shell } from "../utils/shell";
import {
Expand All @@ -17,7 +15,12 @@ import {
releaseFiles,
YargsError
} from "../params";
import { CliGlobalOptions, Compose, Manifest } from "../types";
import { CliGlobalOptions } from "../types";
import { writeReleaseFile } from "../releaseFiles/writeReleaseFile";
import { getReleaseFilePath } from "../releaseFiles/getReleaseFilePath";
import { Compose } from "../releaseFiles/compose/types";
import { Manifest } from "../releaseFiles/manifest/types";
import { ReleaseFileType, AllowedFormats } from "../releaseFiles/types";

const stringsToRemoveFromName = [
"DAppNode-package-",
Expand Down Expand Up @@ -186,7 +189,7 @@ It only covers the most common items, and tries to guess sensible defaults.
};

const compose: Compose = {
version: "3.4",
version: "3.5",
services: {
[serviceName]: {
build: ".", // Dockerfile is in root dir
Expand All @@ -199,7 +202,12 @@ It only covers the most common items, and tries to guess sensible defaults.
// Create package root dir
fs.mkdirSync(dir, { recursive: true });

const manifestPath = getManifestPath(defaultManifestFormat, { dir });
const manifestPath = getReleaseFilePath(
defaultManifestFormat,
ReleaseFileType.manifest,
{ dir }
);

if (fs.existsSync(manifestPath) && !force) {
const continueAnswer = await inquirer.prompt([
{
Expand All @@ -215,11 +223,25 @@ It only covers the most common items, and tries to guess sensible defaults.
}

// Write manifest and compose
writeManifest(manifest, defaultManifestFormat, { dir });
writeReleaseFile(
{ type: ReleaseFileType.manifest, data: manifest },
defaultManifestFormat,
{
dir
}
);

// Only write a compose if it doesn't exist
if (!fs.existsSync(getComposePath({ dir }))) {
writeCompose(compose, { dir, composeFileName });
if (
!fs.existsSync(
getReleaseFilePath(AllowedFormats.yml, ReleaseFileType.compose, { dir })
)
) {
writeReleaseFile(
{ type: ReleaseFileType.compose, data: compose },
AllowedFormats.yml,
{ dir, releaseFileName: composeFileName }
);
}

// Add default avatar so users can run the command right away
Expand Down
36 changes: 10 additions & 26 deletions src/params.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { Architecture, FileFormat, ManifestFormat } from "./types";
import { Architecture } from "./releaseFiles/manifest/types";
import { AllowedFormats, FileFormat } from "./releaseFiles/types";

export class CliError extends Error {}
export class YargsError extends Error {}
Expand All @@ -10,9 +11,16 @@ export const branchNameRoot = "dappnodebot/bump-upstream/";
// DAppNode params

export const defaultDir = "./";
// Manifest
export const defaultManifestFileName = "dappnode_package.json";
export const defaultManifestFormat = ManifestFormat.json;
export const defaultManifestFormat = AllowedFormats.json;
// Compose
export const defaultComposeFileName = "docker-compose.yml";
export const defaultComposeFormat = AllowedFormats.yml;
// Setup Wizard
export const defaultSetupWizardFileName = "setup-wizard.yml";
export const defaultSetupWizardFormat = AllowedFormats.yml;
// Providers
export const publishTxAppUrl = "https://dappnode.github.io/sdk-publish/";
export const UPSTREAM_VERSION_VARNAME = "UPSTREAM_VERSION";
export const upstreamImageLabel = "dappnode.dnp.upstreamImage";
Expand Down Expand Up @@ -57,27 +65,6 @@ export const releaseFiles = {
required: false as const,
multiple: false as const
},
setupSchema: {
regex: /setup\..*\.json$/,
format: FileFormat.JSON,
maxSize: 10e3,
required: false as const,
multiple: false as const
},
setupTarget: {
regex: /setup-target\..*json$/,
format: FileFormat.JSON,
maxSize: 10e3,
required: false as const,
multiple: false as const
},
setupUiJson: {
regex: /setup-ui\..*json$/,
format: FileFormat.JSON,
maxSize: 10e3,
required: false as const,
multiple: false as const
},
disclaimer: {
regex: /disclaimer\.md$/i,
format: FileFormat.TEXT,
Expand Down Expand Up @@ -115,9 +102,6 @@ export const releaseFilesDefaultNames: {
compose: "docker-compose.yml",
avatar: "avatar.png",
setupWizard: "setup-wizard.json",
setupSchema: "setup.schema.json",
setupTarget: "setup-target.json",
setupUiJson: "setup-ui.json",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We cannot merge this change until there's hard proof that no package is using the old setup wizard v1 feature

disclaimer: "disclaimer.md",
gettingStarted: "getting-started.md",
grafanaDashboards: "grafana-dashboard.json",
Expand Down
2 changes: 1 addition & 1 deletion src/pinStrategy/index.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
import { PinataMetadata } from "../releaseUploader/pinata";
import { PinataPinManager } from "../providers/pinata/pinManager";
import { Manifest } from "../types";
import { GitHead } from "../utils/git";
import { prettyPinataPinName } from "../utils/format";
import { DnpPinMetadata, PinDataSummary, PinsByBranch } from "./types";
import { groupPinsByBranch } from "./utils";
import { Manifest } from "../releaseFiles/manifest/types";

// This file has all logic for the pin strategy of DAppNode packages
// In development, builds will be uploaded on each push on a PR
Expand Down
3 changes: 2 additions & 1 deletion src/providers/github/Github.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import path from "path";
import mime from "mime-types";
import retry from "async-retry";
import { Octokit } from "@octokit/rest";
import { getRepoSlugFromManifest } from "../../utils/manifest";
import { getRepoSlugFromManifest } from "../../utils/getRepoSlugFromManifest";

export class Github {
private octokit: Octokit;
Expand Down Expand Up @@ -191,6 +191,7 @@ export class Github {
owner: this.owner,
repo: this.repo,
release_id: release.data.id,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
data: fs.createReadStream(filepath) as any,
headers: {
"content-type": contentType,
Expand Down
Loading