diff --git a/.travis.yml b/.travis.yml index e01679e06b4..a3b694cf20a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ language: node_js sudo: false dist: trusty node_js: - - "6" + - "10" addons: chrome: stable @@ -24,7 +24,8 @@ branches: stages: - test - additional tests - - older version tests + - ember version tests + - external partner tests - name: deploy if: type = push AND (branch IN (master, beta, release) OR tag IS present) @@ -34,18 +35,18 @@ jobs: include: # runs tests with current locked deps and linting - stage: test - env: NAME=test # used only to make Travis UI show description + name: "Basic Tests" script: - ./bin/lint-features - yarn lint:js - yarn test - stage: additional tests - env: NAME=optional-features # used only to make Travis UI show description + name: "Optional Features" install: yarn install script: yarn test:optional-features - - env: NAME=floating dependencies # used only to make Travis UI show description + - name: "Floating Dependencies" install: yarn install --no-lockfile --non-interactive script: yarn test @@ -54,24 +55,48 @@ jobs: env: TARGET_IE11=true script: yarn test - - env: NAME=production # used only to make Travis UI show description + - name: "Production" install: yarn install script: yarn test:production - - env: NAME=node-tests # used only to make Travis UI show description + - name: "Node Tests" install: yarn install script: yarn test:node # runs tests against each supported Ember version - - stage: older version tests + - stage: ember version tests + name: "Ember LTS 2.16" env: EMBER_TRY_SCENARIO=ember-lts-2.16 - - env: EMBER_TRY_SCENARIO=ember-lts-2.18 - - env: EMBER_TRY_SCENARIO=ember-release - - env: EMBER_TRY_SCENARIO=ember-beta - - env: EMBER_TRY_SCENARIO=ember-canary + - name: "Ember LTS 2.18" + env: EMBER_TRY_SCENARIO=ember-lts-2.18 + - name: "Ember Release" + env: EMBER_TRY_SCENARIO=ember-release + - name: "Ember Beta" + env: EMBER_TRY_SCENARIO=ember-beta + - name: "Ember Canary" + env: EMBER_TRY_SCENARIO=ember-canary + + # runs tests against various open-source projects for early-warning regression analysis + - stage: external partner tests + name: "Ember Data Model Fragments" + script: yarn test-external:model-fragments + - name: "Ember Observer" + script: yarn test-external:ember-observer + - name: "Travis Web" + script: yarn test-external:travis-web + - name: "emberaddons.com" + script: yarn test-external:emberaddons.com + - name: "Ember Data Change Tracker" + script: yarn test-external:ember-data-change-tracker + - name: "ember-m3" + script: yarn test-external:ember-m3 + - name: "Ember Data Storefront" + script: yarn test-external:storefront + - name: "Ember Data Factory Guy" + script: yarn test-external:factory-guy - stage: deploy - env: NAME=publish # used only to make Travis UI show description + name: "Publish" install: yarn install script: - node_modules/.bin/ember try:reset @@ -85,6 +110,7 @@ before_install: install: - yarn install + - yarn add global bower script: - node_modules/.bin/ember try:one $EMBER_TRY_SCENARIO --skip-cleanup diff --git a/lib/scripts/test-external.js b/lib/scripts/test-external.js new file mode 100644 index 00000000000..67c2625cdf0 --- /dev/null +++ b/lib/scripts/test-external.js @@ -0,0 +1,116 @@ +'use strict'; +/* eslint-disable no-console, node/no-extraneous-require, node/no-unpublished-require */ +const fs = require('fs'); +const path = require('path'); +const { shellSync } = require('execa'); +// apparently violates no-extraneous require? /shrug +const debug = require('debug')('test-external'); +const rimraf = require('rimraf'); + +const projectRoot = path.resolve(__dirname, '../../'); +const externalProjectName = process.argv[2]; +const gitUrl = process.argv[3]; +const tempDir = path.join(projectRoot, '../__external-test-cache'); +const projectTempDir = path.join(tempDir, externalProjectName); + +if (!gitUrl) { + throw new Error( + 'No git url provided to `node ./lib/scripts/test-external`. An https git url should be the first argument.' + ); +} else if (gitUrl.indexOf('https') !== 0) { + throw new Error( + `The git url provided to \`node ./lib/scripts/test-external\` should use https. Received '${gitUrl}'` + ); +} + +console.log( + `Preparing to test external project ${externalProjectName} located at ${gitUrl} against this ember-data commit.` +); + +function execWithLog(command, force) { + if (debug.enabled || force) { + return shellSync(command, { stdio: [0, 1, 2] }); + } + + return shellSync(command); +} + +if (!fs.existsSync(tempDir)) { + fs.mkdirSync(tempDir); +} + +if (fs.existsSync(projectTempDir)) { + rimraf.sync(projectTempDir); +} + +// install the project +try { + execWithLog( + `cd ../__external-test-cache && git clone --depth=1 ${gitUrl} ./${externalProjectName}` + ); +} catch (e) { + debug(e); + throw new Error( + `Install of ${gitUrl} for external project ${externalProjectName} testing failed.` + ); +} + +const useYarn = fs.existsSync(path.join(projectTempDir, 'yarn.lock')); +const useBower = fs.existsSync(path.join(projectTempDir, 'bower.json')); + +// install project dependencies and link our local version of ember-data +try { + execWithLog( + `${ + useYarn ? 'yarn link' : 'npm link' + } && cd ../__external-test-cache/${externalProjectName} && ${useYarn ? 'yarn' : 'npm install'}${ + useBower ? ' && bower install' : '' + }` + ); +} catch (e) { + debug(e); + throw new Error( + `Unable to complete install of dependencies for external project ${externalProjectName}` + ); +} + +// run project tests +console.log(`Running tests for ${externalProjectName}`); + +let smokeTestPassed = true; +let commitTestPassed = true; + +try { + debug('Running Smoke Test'); + execWithLog(`cd ../__external-test-cache/${externalProjectName} && ember test`); +} catch (e) { + smokeTestPassed = false; +} + +try { + execWithLog(`${useYarn ? 'yarn link ember-data' : 'npm link ember-data'}`); +} catch (e) { + debug(e); + throw new Error( + `Unable to \`${useYarn ? 'yarn' : 'npm'} link ember-data\` for ${externalProjectName}` + ); +} + +try { + debug('Re-running tests against EmberData commit'); + execWithLog(`cd ../__external-test-cache/${externalProjectName} && ember test`, true); +} catch (e) { + commitTestPassed = false; +} + +if (!smokeTestPassed && !commitTestPassed) { + throw new Error( + `Commit may result in a regression, but the smoke test for ${externalProjectName} also failed.` + ); +} else if (smokeTestPassed && !commitTestPassed) { + throw new Error(`Commit results in a regression in ${externalProjectName}`); +} else if (!smokeTestPassed) { + console.log(`Commit may resolve issues present in the smoke test for ${externalProjectName}`); +} else { + console.log(`Commit does not regress ${externalProjectName}`); +} diff --git a/package.json b/package.json index 2c77a780595..6d998d9ba94 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,15 @@ "test:all": "ember try:each", "test:node": "node node-tests/nodetest-runner.js", "test:production": "ember test -e production && ember test -e production --record-data-rfc-build", - "test:optional-features": "ember test -e test-optional-features && ember test -e test-optional-features --record-data-rfc-build" + "test:optional-features": "ember test -e test-optional-features && ember test -e test-optional-features --record-data-rfc-build", + "test-external:ember-m3": "node ./lib/scripts/test-external ember-m3 https://github.com/hjdivad/ember-m3.git", + "test-external:ember-data-change-tracker": "node ./lib/scripts/test-external ember-data-change-tracker https://github.com/danielspaniel/ember-data-change-tracker.git", + "test-external:emberaddons.com": "node ./lib/scripts/test-external ember-cli-addon-search https://github.com/gcollazo/ember-cli-addon-search.git", + "test-external:model-fragments": "node ./lib/scripts/test-external ember-data-model-fragments https://github.com/lytics/ember-data-model-fragments.git", + "test-external:ember-observer": "node ./lib/scripts/test-external ember-observer https://github.com/emberobserver/client.git", + "test-external:travis-web": "node ./lib/scripts/test-external travis-web https://github.com/travis-ci/travis-web.git", + "test-external:storefront": "node ./lib/scripts/test-external storefront https://github.com/embermap/ember-data-storefront.git", + "test-external:factory-guy": "node ./lib/scripts/test-external factory-guy https://github.com/danielspaniel/ember-data-factory-guy.git" }, "author": "", "license": "MIT", @@ -92,6 +100,7 @@ "eslint-config-prettier": "^2.9.0", "eslint-plugin-node": "^7.0.1", "eslint-plugin-prettier": "2.6.2", + "execa": "^0.10.0", "fs-extra": "^7.0.0", "github": "^1.1.1", "glob": "^7.1.2", diff --git a/yarn.lock b/yarn.lock index f4bc2465fb9..ed542ce154f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3157,7 +3157,7 @@ exec-sh@^0.2.0: execa@^0.10.0: version "0.10.0" - resolved "https://registry.yarnpkg.com/execa/-/execa-0.10.0.tgz#ff456a8f53f90f8eccc71a96d11bdfc7f082cb50" + resolved "https://registry.npmjs.org/execa/-/execa-0.10.0.tgz#ff456a8f53f90f8eccc71a96d11bdfc7f082cb50" dependencies: cross-spawn "^6.0.0" get-stream "^3.0.0"