Re-implement GitHub URL parsing, fix related bugs

packages_rs/nextclade-web/package.json

@@ -297,9 +297,10 @@
     "i18next-parser": "6.3.0",
     "identity-obj-proxy": "3.0.0",
     "is-interactive": "1.0.0",
-    "jest": "28.0.1",
+    "jest": "27.5.1",
     "jest-axe": "6.0.0",
-    "jest-chain": "1.1.5",
+    "jest-chain": "1.1.6",
+    "jest-environment-jsdom": "27.5.1",
     "jest-extended": "2.0.0",
     "jest-raw-loader": "1.0.1",
     "jest-runner-eslint": "1.0.0",

packages_rs/nextclade-web/src/components/Error/ErrorContent.tsx

@@ -43,7 +43,7 @@ export function ErrorContentMessage({ error }: { error: Error }) {
     const url = error.url ?? 'Unknown URL'
     const { status, statusText, message } = error
 
-    if (!status) {
+    if (!status || status === 'ERR_BAD_REQUEST') {
       return <ErrorNetworkConnectionFailure url={url} message={message} />
     }
     const text = message ?? statusText ?? 'Unknown status'

packages_rs/nextclade-web/src/constants.ts

@@ -33,6 +33,9 @@ export const URL_CLADE_SCHEMA_REPO = 'https://github.com/nextstrain/ncov-clades-
 export const URL_CLADE_SCHEMA_SVG = 'https://raw.githubusercontent.com/nextstrain/ncov-clades-schema/master/clades.svg'
 
 export const URL_GITHUB_DATA_RAW = 'https://raw.githubusercontent.com/nextstrain/nextclade_data' as const
+export const DEFAULT_DATA_OWNER = 'nextstrain' as const
+export const DEFAULT_DATA_REPO = 'nextclade_data' as const
+export const DEFAULT_DATA_REPO_PATH = 'data_output' as const
 
 export const SUPPORT_EMAIL = '[email protected]'
 

packages_rs/nextclade-web/src/helpers/ErrorFatal.ts

@@ -0,0 +1,5 @@
+export class ErrorFatal extends Error {
+  public constructor(message: string) {
+    super(message)
+  }
+}

packages_rs/nextclade-web/src/helpers/jestUtils.ts

@@ -0,0 +1,3 @@
+export function getTestName(): string {
+  return expect.getState().currentTestName.split(' ')[1]
+}

packages_rs/nextclade-web/src/io/__tests__/getDatasetServerUrl.test.ts

@@ -0,0 +1,78 @@
+import { getDatasetServerUrl } from 'src/io/fetchDatasets'
+
+describe('parseGitHubRepoShortcut', () => {
+  const OLD_ENV = process.env
+
+  beforeEach(() => {
+    jest.resetModules()
+    process.env = { ...OLD_ENV, BRANCH_NAME: 'default-branch' }
+  })
+
+  afterAll(() => {
+    process.env = OLD_ENV
+  })
+
+  it.each([
+    ['gh', 'https://raw.githubusercontent.com/nextstrain/nextclade_data/default-branch/data_output'],
+    ['gh:', 'https://raw.githubusercontent.com/nextstrain/nextclade_data/default-branch/data_output'],
+    ['gh', 'https://raw.githubusercontent.com/nextstrain/nextclade_data/default-branch/data_output'],
+    ['gh:', 'https://raw.githubusercontent.com/nextstrain/nextclade_data/default-branch/data_output'],
+    ['gh:@test-branch@', 'https://raw.githubusercontent.com/nextstrain/nextclade_data/test-branch/data_output'],
+    ['gh:aaa/bbb', 'https://raw.githubusercontent.com/aaa/bbb/master/'],
+    ['gh:aaa/bbb/', 'https://raw.githubusercontent.com/aaa/bbb/master/'],
+    ['gh:aaa/bbb@test-branch@', 'https://raw.githubusercontent.com/aaa/bbb/test-branch/'],
+    ['gh:aaa/bbb@test-branch@my/path/dir', 'https://raw.githubusercontent.com/aaa/bbb/test-branch/my/path/dir'],
+    ['gh:aaa/bbb@test-branch@/my/path/dir', 'https://raw.githubusercontent.com/aaa/bbb/test-branch/my/path/dir'],
+    ['gh:aaa/bbb@test-branch@my/path/dir/', 'https://raw.githubusercontent.com/aaa/bbb/test-branch/my/path/dir'],
+    ['gh:aaa/bbb@test-branch@/my/path/dir/', 'https://raw.githubusercontent.com/aaa/bbb/test-branch/my/path/dir'],
+    ['gh:aaa/bbb@branch/slashes@/my/path/dir', 'https://raw.githubusercontent.com/aaa/bbb/branch/slashes/my/path/dir'],
+    ['https://github.com/aaa/bbb', 'https://raw.githubusercontent.com/aaa/bbb/master/'],
+    ['https://github.com/aaa/bbb/', 'https://raw.githubusercontent.com/aaa/bbb/master/'],
+    ['https://github.com/aaa/bbb/tree/test-branch', 'https://raw.githubusercontent.com/aaa/bbb/test-branch/'],
+    ['https://github.com/aaa/bbb/tree/test-branch/', 'https://raw.githubusercontent.com/aaa/bbb/test-branch/'],
+    [
+      'https://github.com/aaa/bbb/tree/test-branch/dirname',
+      'https://raw.githubusercontent.com/aaa/bbb/test-branch/dirname',
+    ],
+    [
+      'https://github.com/aaa/bbb/tree/test-branch/dirname/',
+      'https://raw.githubusercontent.com/aaa/bbb/test-branch/dirname',
+    ],
+    [
+      'https://github.com/aaa/bbb/tree/test-branch/dirname//',
+      'https://raw.githubusercontent.com/aaa/bbb/test-branch/dirname',
+    ],
+    ['https://github.com/aaa/bbb/blob/test-branch', 'https://raw.githubusercontent.com/aaa/bbb/test-branch/'],
+    ['https://github.com/aaa/bbb/blob/test-branch/', 'https://raw.githubusercontent.com/aaa/bbb/test-branch/'],
+    [
+      'https://github.com/aaa/bbb/blob/test-branch/dirname',
+      'https://raw.githubusercontent.com/aaa/bbb/test-branch/dirname',
+    ],
+    [
+      'https://github.com/aaa/bbb/blob/test-branch/my/path/dir/',
+      'https://raw.githubusercontent.com/aaa/bbb/test-branch/my/path/dir',
+    ],
+    [
+      'https://github.com/aaa/bbb/blob/test-branch/dirname/filename.json',
+      'https://raw.githubusercontent.com/aaa/bbb/test-branch/dirname/filename.json',
+    ],
+    [
+      'https://github.com/aaa/bbb/blob/branch/slashes/dirname/filename.json',
+      'https://raw.githubusercontent.com/aaa/bbb/branch/slashes/dirname/filename.json',
+    ],
+    [
+      'https://github.com/aaa/bbb/blob/test-branch/dirname//',
+      'https://raw.githubusercontent.com/aaa/bbb/test-branch/dirname',
+    ],
+    [
+      'https://github.com/nextstrain/nextclade_data/tree/master/data_output/nextstrain/rsv/a/EPI_ISL_412866/unreleased',
+      'https://raw.githubusercontent.com/nextstrain/nextclade_data/master/data_output/nextstrain/rsv/a/EPI_ISL_412866/unreleased',
+    ],
+    [
+      'https://github.com/nextstrain/nextclade_data/tree/release/data_output/nextstrain/rsv/a/EPI_ISL_412866/unreleased',
+      'https://raw.githubusercontent.com/nextstrain/nextclade_data/release/data_output/nextstrain/rsv/a/EPI_ISL_412866/unreleased',
+    ],
+  ])('%p', async (input: string, result: string) => {
+    expect(await getDatasetServerUrl({ 'dataset-server': input })).toBe(result)
+  })
+})

packages_rs/nextclade-web/src/io/__tests__/parseGitHubRepoShortcut.test.ts

@@ -0,0 +1,106 @@
+import { DEFAULT_DATA_OWNER, DEFAULT_DATA_REPO, DEFAULT_DATA_REPO_PATH } from 'src/constants'
+import { getTestName } from 'src/helpers/jestUtils'
+import { parseGitHubRepoShortcut } from 'src/io/fetchSingleDatasetFromGithub'
+
+describe('parseGitHubRepoShortcut', () => {
+  const OLD_ENV = process.env
+
+  beforeEach(() => {
+    jest.resetModules()
+    process.env = { ...OLD_ENV, BRANCH_NAME: 'default-branch' }
+  })
+
+  afterAll(() => {
+    process.env = OLD_ENV
+  })
+
+  it('gh', async () => {
+    expect(await parseGitHubRepoShortcut(getTestName())).toStrictEqual({
+      owner: DEFAULT_DATA_OWNER,
+      repo: DEFAULT_DATA_REPO,
+      branch: 'default-branch',
+      path: DEFAULT_DATA_REPO_PATH,
+    })
+  })
+
+  it('gh:', async () => {
+    expect(await parseGitHubRepoShortcut(getTestName())).toStrictEqual({
+      owner: DEFAULT_DATA_OWNER,
+      repo: DEFAULT_DATA_REPO,
+      branch: 'default-branch',
+      path: DEFAULT_DATA_REPO_PATH,
+    })
+  })
+
+  it('gh:@test-branch@', async () => {
+    expect(await parseGitHubRepoShortcut(getTestName())).toStrictEqual({
+      owner: DEFAULT_DATA_OWNER,
+      repo: DEFAULT_DATA_REPO,
+      branch: 'test-branch',
+      path: DEFAULT_DATA_REPO_PATH,
+    })
+  })
+
+  it('gh:aaa/bbb', async () => {
+    expect(await parseGitHubRepoShortcut(getTestName())).toStrictEqual({
+      owner: 'aaa',
+      repo: 'bbb',
+      branch: 'default-branch',
+      path: '/',
+    })
+  })
+
+  it('gh:aaa/bbb/', async () => {
+    expect(await parseGitHubRepoShortcut(getTestName())).toStrictEqual({
+      owner: 'aaa',
+      repo: 'bbb',
+      branch: 'default-branch',
+      path: '/',
+    })
+  })
+
+  it('gh:aaa/bbb@test-branch@', async () => {
+    expect(await parseGitHubRepoShortcut(getTestName())).toStrictEqual({
+      owner: 'aaa',
+      repo: 'bbb',
+      branch: 'test-branch',
+      path: '/',
+    })
+  })
+
+  it('gh:aaa/bbb@test-branch@my/path/dir', async () => {
+    expect(await parseGitHubRepoShortcut(getTestName())).toStrictEqual({
+      owner: 'aaa',
+      repo: 'bbb',
+      branch: 'test-branch',
+      path: 'my/path/dir',
+    })
+  })
+
+  it('gh:aaa/bbb@test-branch@/my/path/dir', async () => {
+    expect(await parseGitHubRepoShortcut(getTestName())).toStrictEqual({
+      owner: 'aaa',
+      repo: 'bbb',
+      branch: 'test-branch',
+      path: 'my/path/dir',
+    })
+  })
+
+  it('gh:aaa/bbb@test-branch@my/path/dir/', async () => {
+    expect(await parseGitHubRepoShortcut(getTestName())).toStrictEqual({
+      owner: 'aaa',
+      repo: 'bbb',
+      branch: 'test-branch',
+      path: 'my/path/dir',
+    })
+  })
+
+  it('gh:aaa/bbb@test-branch@/my/path/dir/', async () => {
+    expect(await parseGitHubRepoShortcut(getTestName())).toStrictEqual({
+      owner: 'aaa',
+      repo: 'bbb',
+      branch: 'test-branch',
+      path: 'my/path/dir',
+    })
+  })
+})

packages_rs/nextclade-web/src/io/__tests__/parseGitHubRepoShortcutComponents.test.ts

@@ -0,0 +1,148 @@
+import { getTestName } from 'src/helpers/jestUtils'
+import { parseGitHubRepoShortcutComponents } from 'src/io/fetchSingleDatasetFromGithub'
+
+describe('parseGitHubRepoShortcutComponents', () => {
+  it('gh', async () => {
+    expect(parseGitHubRepoShortcutComponents(getTestName())).toStrictEqual({
+      owner: undefined,
+      repo: undefined,
+      branch: undefined,
+      path: undefined,
+    })
+  })
+
+  it('gh:', async () => {
+    expect(parseGitHubRepoShortcutComponents(getTestName())).toStrictEqual({
+      owner: undefined,
+      repo: undefined,
+      branch: undefined,
+      path: undefined,
+    })
+  })
+
+  it('gh:@test-branch@', async () => {
+    expect(parseGitHubRepoShortcutComponents(getTestName())).toStrictEqual({
+      owner: undefined,
+      repo: undefined,
+      branch: 'test-branch',
+      path: undefined,
+    })
+  })
+
+  it('gh:aaa/bbb', async () => {
+    expect(parseGitHubRepoShortcutComponents(getTestName())).toStrictEqual({
+      owner: 'aaa',
+      repo: 'bbb',
+      branch: undefined,
+      path: undefined,
+    })
+  })
+
+  it('gh:aaa/bbb/', async () => {
+    expect(parseGitHubRepoShortcutComponents(getTestName())).toStrictEqual({
+      owner: 'aaa',
+      repo: 'bbb',
+      branch: undefined,
+      path: '/',
+    })
+  })
+
+  it('gh:aaa/bbb@test-branch@', async () => {
+    expect(parseGitHubRepoShortcutComponents(getTestName())).toStrictEqual({
+      owner: 'aaa',
+      repo: 'bbb',
+      branch: 'test-branch',
+      path: undefined,
+    })
+  })
+
+  it('gh:aaa/bbb@branch/slashes@', async () => {
+    expect(parseGitHubRepoShortcutComponents(getTestName())).toStrictEqual({
+      owner: 'aaa',
+      repo: 'bbb',
+      branch: 'branch/slashes',
+      path: undefined,
+    })
+  })
+
+  it('gh:aaa/bbb@branch/more/slashes@', async () => {
+    expect(parseGitHubRepoShortcutComponents(getTestName())).toStrictEqual({
+      owner: 'aaa',
+      repo: 'bbb',
+      branch: 'branch/more/slashes',
+      path: undefined,
+    })
+  })
+
+  it('gh:aaa/bbb@test-branch@my/path/dir', async () => {
+    expect(parseGitHubRepoShortcutComponents(getTestName())).toStrictEqual({
+      owner: 'aaa',
+      repo: 'bbb',
+      branch: 'test-branch',
+      path: 'my/path/dir',
+    })
+  })
+
+  it('gh:aaa/bbb@test-branch@/my/path/dir', async () => {
+    expect(parseGitHubRepoShortcutComponents(getTestName())).toStrictEqual({
+      owner: 'aaa',
+      repo: 'bbb',
+      branch: 'test-branch',
+      path: 'my/path/dir',
+    })
+  })
+
+  it('gh:aaa/bbb@branch/slashes@/my/path/dir', async () => {
+    expect(parseGitHubRepoShortcutComponents(getTestName())).toStrictEqual({
+      owner: 'aaa',
+      repo: 'bbb',
+      branch: 'branch/slashes',
+      path: 'my/path/dir',
+    })
+  })
+
+  it('gh:aaa/bbb@branch/more/slashes@/my/path/dir', async () => {
+    expect(parseGitHubRepoShortcutComponents(getTestName())).toStrictEqual({
+      owner: 'aaa',
+      repo: 'bbb',
+      branch: 'branch/more/slashes',
+      path: 'my/path/dir',
+    })
+  })
+
+  it('gh:aaa/bbb@test-branch@my/path/dir/', async () => {
+    expect(parseGitHubRepoShortcutComponents(getTestName())).toStrictEqual({
+      owner: 'aaa',
+      repo: 'bbb',
+      branch: 'test-branch',
+      path: 'my/path/dir',
+    })
+  })
+
+  it('gh:aaa/bbb@test-branch@/my/path/dir/', async () => {
+    expect(parseGitHubRepoShortcutComponents(getTestName())).toStrictEqual({
+      owner: 'aaa',
+      repo: 'bbb',
+      branch: 'test-branch',
+      path: 'my/path/dir',
+    })
+  })
+
+  it('gh:aaa/bbb/my/path/dir', async () => {
+    expect(parseGitHubRepoShortcutComponents(getTestName())).toStrictEqual({
+      owner: 'aaa',
+      repo: 'bbb',
+      branch: undefined,
+      path: 'my/path/dir',
+    })
+  })
+
+  it('gh:aaa/bbb/my/path/dir/', async () => {
+    expect(parseGitHubRepoShortcutComponents(getTestName())).toStrictEqual({
+      owner: 'aaa',
+      repo: 'bbb',
+      branch: undefined,
+      path: 'my/path/dir',
+    })
+  })
+})