Skip to content

Commit

Permalink
enable external partner testing
Browse files Browse the repository at this point in the history
add names to external partner tests

use shallow cloning, use bower if necessary, add more external partners

nice names

smoke tests

prettier and storefront

use execa

fix bower command

use node 10 in travis
  • Loading branch information
runspired committed Aug 15, 2018
1 parent 3a5632f commit cb9d740
Show file tree
Hide file tree
Showing 4 changed files with 166 additions and 15 deletions.
52 changes: 39 additions & 13 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ language: node_js
sudo: false
dist: trusty
node_js:
- "6"
- "10"

addons:
chrome: stable
Expand All @@ -24,7 +24,8 @@ branches:
stages:
- test
- additional tests
- older version tests
- ember version tests
- external partner tests
- name: deploy
if: type = push AND (branch IN (master, beta, release) OR tag IS present)

Expand All @@ -34,18 +35,18 @@ jobs:
include:
# runs tests with current locked deps and linting
- stage: test
env: NAME=test # used only to make Travis UI show description
name: "Basic Tests"
script:
- ./bin/lint-features
- yarn lint:js
- yarn test

- stage: additional tests
env: NAME=optional-features # used only to make Travis UI show description
name: "Optional Features"
install: yarn install
script: yarn test:optional-features

- env: NAME=floating dependencies # used only to make Travis UI show description
- name: "Floating Dependencies"
install: yarn install --no-lockfile --non-interactive
script: yarn test

Expand All @@ -54,24 +55,48 @@ jobs:
env: TARGET_IE11=true
script: yarn test

- env: NAME=production # used only to make Travis UI show description
- name: "Production"
install: yarn install
script: yarn test:production

- env: NAME=node-tests # used only to make Travis UI show description
- name: "Node Tests"
install: yarn install
script: yarn test:node

# runs tests against each supported Ember version
- stage: older version tests
- stage: ember version tests
name: "Ember LTS 2.16"
env: EMBER_TRY_SCENARIO=ember-lts-2.16
- env: EMBER_TRY_SCENARIO=ember-lts-2.18
- env: EMBER_TRY_SCENARIO=ember-release
- env: EMBER_TRY_SCENARIO=ember-beta
- env: EMBER_TRY_SCENARIO=ember-canary
- name: "Ember LTS 2.18"
env: EMBER_TRY_SCENARIO=ember-lts-2.18
- name: "Ember Release"
env: EMBER_TRY_SCENARIO=ember-release
- name: "Ember Beta"
env: EMBER_TRY_SCENARIO=ember-beta
- name: "Ember Canary"
env: EMBER_TRY_SCENARIO=ember-canary

# runs tests against various open-source projects for early-warning regression analysis
- stage: external partner tests
name: "Ember Data Model Fragments"
script: yarn test-external:model-fragments
- name: "Ember Observer"
script: yarn test-external:ember-observer
- name: "Travis Web"
script: yarn test-external:travis-web
- name: "emberaddons.com"
script: yarn test-external:emberaddons.com
- name: "Ember Data Change Tracker"
script: yarn test-external:ember-data-change-tracker
- name: "ember-m3"
script: yarn test-external:ember-m3
- name: "Ember Data Storefront"
script: yarn test-external:storefront
- name: "Ember Data Factory Guy"
script: yarn test-external:factory-guy

- stage: deploy
env: NAME=publish # used only to make Travis UI show description
name: "Publish"
install: yarn install
script:
- node_modules/.bin/ember try:reset
Expand All @@ -85,6 +110,7 @@ before_install:

install:
- yarn install
- yarn add global bower

script:
- node_modules/.bin/ember try:one $EMBER_TRY_SCENARIO --skip-cleanup
Expand Down
116 changes: 116 additions & 0 deletions lib/scripts/test-external.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
'use strict';
/* eslint-disable no-console, node/no-extraneous-require, node/no-unpublished-require */
const fs = require('fs');
const path = require('path');
const { shellSync } = require('execa');
// apparently violates no-extraneous require? /shrug
const debug = require('debug')('test-external');
const rimraf = require('rimraf');

const projectRoot = path.resolve(__dirname, '../../');
const externalProjectName = process.argv[2];
const gitUrl = process.argv[3];
const tempDir = path.join(projectRoot, '../__external-test-cache');
const projectTempDir = path.join(tempDir, externalProjectName);

if (!gitUrl) {
throw new Error(
'No git url provided to `node ./lib/scripts/test-external`. An https git url should be the first argument.'
);
} else if (gitUrl.indexOf('https') !== 0) {
throw new Error(
`The git url provided to \`node ./lib/scripts/test-external\` should use https. Received '${gitUrl}'`
);
}

console.log(
`Preparing to test external project ${externalProjectName} located at ${gitUrl} against this ember-data commit.`
);

function execWithLog(command, force) {
if (debug.enabled || force) {
return shellSync(command, { stdio: [0, 1, 2] });
}

return shellSync(command);
}

if (!fs.existsSync(tempDir)) {
fs.mkdirSync(tempDir);
}

if (fs.existsSync(projectTempDir)) {
rimraf.sync(projectTempDir);
}

// install the project
try {
execWithLog(
`cd ../__external-test-cache && git clone --depth=1 ${gitUrl} ./${externalProjectName}`
);
} catch (e) {
debug(e);
throw new Error(
`Install of ${gitUrl} for external project ${externalProjectName} testing failed.`
);
}

const useYarn = fs.existsSync(path.join(projectTempDir, 'yarn.lock'));
const useBower = fs.existsSync(path.join(projectTempDir, 'bower.json'));

// install project dependencies and link our local version of ember-data
try {
execWithLog(
`${
useYarn ? 'yarn link' : 'npm link'
} && cd ../__external-test-cache/${externalProjectName} && ${useYarn ? 'yarn' : 'npm install'}${
useBower ? ' && bower install' : ''
}`
);
} catch (e) {
debug(e);
throw new Error(
`Unable to complete install of dependencies for external project ${externalProjectName}`
);
}

// run project tests
console.log(`Running tests for ${externalProjectName}`);

let smokeTestPassed = true;
let commitTestPassed = true;

try {
debug('Running Smoke Test');
execWithLog(`cd ../__external-test-cache/${externalProjectName} && ember test`);
} catch (e) {
smokeTestPassed = false;
}

try {
execWithLog(`${useYarn ? 'yarn link ember-data' : 'npm link ember-data'}`);
} catch (e) {
debug(e);
throw new Error(
`Unable to \`${useYarn ? 'yarn' : 'npm'} link ember-data\` for ${externalProjectName}`
);
}

try {
debug('Re-running tests against EmberData commit');
execWithLog(`cd ../__external-test-cache/${externalProjectName} && ember test`, true);
} catch (e) {
commitTestPassed = false;
}

if (!smokeTestPassed && !commitTestPassed) {
throw new Error(
`Commit may result in a regression, but the smoke test for ${externalProjectName} also failed.`
);
} else if (smokeTestPassed && !commitTestPassed) {
throw new Error(`Commit results in a regression in ${externalProjectName}`);
} else if (!smokeTestPassed) {
console.log(`Commit may resolve issues present in the smoke test for ${externalProjectName}`);
} else {
console.log(`Commit does not regress ${externalProjectName}`);
}
11 changes: 10 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,15 @@
"test:all": "ember try:each",
"test:node": "node node-tests/nodetest-runner.js",
"test:production": "ember test -e production && ember test -e production --record-data-rfc-build",
"test:optional-features": "ember test -e test-optional-features && ember test -e test-optional-features --record-data-rfc-build"
"test:optional-features": "ember test -e test-optional-features && ember test -e test-optional-features --record-data-rfc-build",
"test-external:ember-m3": "node ./lib/scripts/test-external ember-m3 https://github.com/hjdivad/ember-m3.git",
"test-external:ember-data-change-tracker": "node ./lib/scripts/test-external ember-data-change-tracker https://github.com/danielspaniel/ember-data-change-tracker.git",
"test-external:emberaddons.com": "node ./lib/scripts/test-external ember-cli-addon-search https://github.com/gcollazo/ember-cli-addon-search.git",
"test-external:model-fragments": "node ./lib/scripts/test-external ember-data-model-fragments https://github.com/lytics/ember-data-model-fragments.git",
"test-external:ember-observer": "node ./lib/scripts/test-external ember-observer https://github.com/emberobserver/client.git",
"test-external:travis-web": "node ./lib/scripts/test-external travis-web https://github.com/travis-ci/travis-web.git",
"test-external:storefront": "node ./lib/scripts/test-external storefront https://github.com/embermap/ember-data-storefront.git",
"test-external:factory-guy": "node ./lib/scripts/test-external factory-guy https://github.com/danielspaniel/ember-data-factory-guy.git"
},
"author": "",
"license": "MIT",
Expand Down Expand Up @@ -92,6 +100,7 @@
"eslint-config-prettier": "^2.9.0",
"eslint-plugin-node": "^7.0.1",
"eslint-plugin-prettier": "2.6.2",
"execa": "^0.10.0",
"fs-extra": "^7.0.0",
"github": "^1.1.1",
"glob": "^7.1.2",
Expand Down
2 changes: 1 addition & 1 deletion yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -3157,7 +3157,7 @@ exec-sh@^0.2.0:

execa@^0.10.0:
version "0.10.0"
resolved "https://registry.yarnpkg.com/execa/-/execa-0.10.0.tgz#ff456a8f53f90f8eccc71a96d11bdfc7f082cb50"
resolved "https://registry.npmjs.org/execa/-/execa-0.10.0.tgz#ff456a8f53f90f8eccc71a96d11bdfc7f082cb50"
dependencies:
cross-spawn "^6.0.0"
get-stream "^3.0.0"
Expand Down

0 comments on commit cb9d740

Please sign in to comment.