diff --git a/README.md b/README.md index a7f6f84..f44045f 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,9 @@ USAGE * [`heroku ai:models:attach MODEL_RESOURCE`](#heroku-aimodelsattach-model_resource) * [`heroku ai:models:call MODEL_RESOURCE`](#heroku-aimodelscall-model_resource) * [`heroku ai:models:create MODEL_NAME`](#heroku-aimodelscreate-model_name) +* [`heroku ai:models:destroy MODELRESOURCE`](#heroku-aimodelsdestroy-modelresource) +* [`heroku ai:models:detach MODEL_RESOURCE`](#heroku-aimodelsdetach-model_resource) +* [`heroku ai:models:info [MODELRESOURCE]`](#heroku-aimodelsinfo-modelresource) * [`heroku ai:models:list`](#heroku-aimodelslist) ## `heroku ai:docs` @@ -157,6 +160,82 @@ EXAMPLES _See code: [dist/commands/ai/models/create.ts](https://github.com/heroku/heroku-cli-plugin-integration/blob/v0.0.0/dist/commands/ai/models/create.ts)_ +## `heroku ai:models:destroy MODELRESOURCE` + +destroy an existing AI model resource + +``` +USAGE + $ heroku ai:models:destroy [MODELRESOURCE] -a [-c ] [-f] [-r ] + +ARGUMENTS + MODELRESOURCE The resource ID or alias of the model resource to destroy. + +FLAGS + -a, --app= (required) app to run command against + -c, --confirm= + -f, --force allow destruction even if connected to other apps + -r, --remote= git remote of app to use + +DESCRIPTION + destroy an existing AI model resource + +EXAMPLES + $ heroku ai:models:destroy claude-3-5-sonnet-acute-43973 +``` + +_See code: [dist/commands/ai/models/destroy.ts](https://github.com/heroku/heroku-cli-plugin-integration/blob/v0.0.0/dist/commands/ai/models/destroy.ts)_ + +## `heroku ai:models:detach MODEL_RESOURCE` + +Detach a model resource from an app. + +``` +USAGE + $ heroku ai:models:detach [MODEL_RESOURCE] -a [-r ] + +ARGUMENTS + MODEL_RESOURCE The resource ID or alias of the model resource to detach + +FLAGS + -a, --app= (required) The name of the Heroku app to detach the model resource from. + -r, --remote= git remote of app to use + +DESCRIPTION + Detach a model resource from an app. + +EXAMPLES + $ heroku ai:models:detach claude-3-5-sonnet-acute-41518 --app example-app +``` + +_See code: [dist/commands/ai/models/detach.ts](https://github.com/heroku/heroku-cli-plugin-integration/blob/v0.0.0/dist/commands/ai/models/detach.ts)_ + +## `heroku ai:models:info [MODELRESOURCE]` + +get the current status of all the AI model resources attached to your app or a specific resource + +``` +USAGE + $ heroku ai:models:info [MODELRESOURCE] -a [-r ] + +ARGUMENTS + MODELRESOURCE The resource ID or alias of the model resource to check. + +FLAGS + -a, --app= (required) app to run command against + -r, --remote= git remote of app to use + +DESCRIPTION + get the current status of all the AI model resources attached to your app or a specific resource + +EXAMPLES + $ heroku ai:models:info claude-3-5-sonnet-acute-04281 --app example-app + + $ heroku ai:models:info --app example-app +``` + +_See code: [dist/commands/ai/models/info.ts](https://github.com/heroku/heroku-cli-plugin-integration/blob/v0.0.0/dist/commands/ai/models/info.ts)_ + ## `heroku ai:models:list` list available AI models to provision access to diff --git a/package.json b/package.json index b4bf6d3..43c29dd 100644 --- a/package.json +++ b/package.json @@ -71,6 +71,7 @@ "lint": "eslint . --ext .ts --config .eslintrc.json", "posttest": "yarn lint", "test": "nyc mocha --forbid-only", + "test:local": "nyc mocha", "version": "oclif readme && git add README.md" } } diff --git a/src/commands/ai/models/destroy.ts b/src/commands/ai/models/destroy.ts new file mode 100644 index 0000000..63be467 --- /dev/null +++ b/src/commands/ai/models/destroy.ts @@ -0,0 +1,38 @@ +import {Args} from '@oclif/core' +import {flags} from '@heroku-cli/command' +import destroyAddon from '../../../lib/ai/models/destroy_addon' +import confirmCommand from '../../../lib/confirmCommand' +import Command from '../../../lib/base' + +export default class Destroy extends Command { + static description = 'destroy an existing AI model resource' + + static flags = { + app: flags.app({required: true, description: 'app to run command against'}), + confirm: flags.string({char: 'c'}), + force: flags.boolean({char: 'f', description: 'allow destruction even if connected to other apps'}), + remote: flags.remote({description: 'git remote of app to use'}), + } + + static args = { + modelResource: Args.string({required: true, description: 'The resource ID or alias of the model resource to destroy.'}), + } + + static examples = [ + '$ heroku ai:models:destroy claude-3-5-sonnet-acute-43973', + ] + + public async run(): Promise { + const {flags, args} = await this.parse(Destroy) + const {app, confirm} = flags + const {modelResource} = args + const force = flags.force || process.env.HEROKU_FORCE === '1' + + await this.configureHerokuAIClient(modelResource, app) + + const aiAddon = this.addon + + await confirmCommand(app, confirm) + await destroyAddon(this.config, aiAddon, force) + } +} diff --git a/src/commands/ai/models/detach.ts b/src/commands/ai/models/detach.ts new file mode 100644 index 0000000..667a836 --- /dev/null +++ b/src/commands/ai/models/detach.ts @@ -0,0 +1,46 @@ +import color from '@heroku-cli/color' +import {flags} from '@heroku-cli/command' +import {Args, ux} from '@oclif/core' +import * as Heroku from '@heroku-cli/schema' +import Command from '../../../lib/base' + +export default class Detach extends Command { + static description = 'Detach a model resource from an app.' + static flags = { + app: flags.app({description: 'The name of the Heroku app to detach the model resource from.', required: true}), + remote: flags.remote(), + } + + static args = { + model_resource: Args.string({ + description: 'The resource ID or alias of the model resource to detach', + required: true, + }), + } + + static example = 'heroku ai:models:detach claude-3-5-sonnet-acute-41518 --app example-app' + + public async run(): Promise { + const {flags, args} = await this.parse(Detach) + const {app} = flags + const {model_resource: modelResource} = args + + await this.configureHerokuAIClient(modelResource, app) + + const aiAddon = this.addon + + ux.action.start(`Detaching ${color.cyan(aiAddon.name || '')} from ${color.magenta(app)}`) + + await this.heroku.delete(`/addon-attachments/${aiAddon.id}`) + + ux.action.stop() + + ux.action.start(`Unsetting ${color.cyan(aiAddon.name || '')} config vars and restarting ${color.magenta(app)}`) + + const {body: releases} = await this.heroku.get(`/apps/${app}/releases`, { + partial: true, headers: {Range: 'version ..; max=1, order=desc'}, + }) + + ux.action.stop(`done, v${releases[0]?.version || ''}`) + } +} diff --git a/src/commands/ai/models/info.ts b/src/commands/ai/models/info.ts new file mode 100644 index 0000000..964cf51 --- /dev/null +++ b/src/commands/ai/models/info.ts @@ -0,0 +1,105 @@ +import color from '@heroku-cli/color' +import {flags} from '@heroku-cli/command' +import {Args, ux} from '@oclif/core' +import Command from '../../../lib/base' +import {ModelResource} from '../../../lib/ai/types' +import appAddons from '../../../lib/ai/models/app_addons' +import * as Heroku from '@heroku-cli/schema' + +export default class Info extends Command { + static description = 'get the current status of all the AI model resources attached to your app or a specific resource' + static examples = [ + 'heroku ai:models:info claude-3-5-sonnet-acute-04281 --app example-app', + 'heroku ai:models:info --app example-app', + ] + + static flags = { + app: flags.app({required: true}), + remote: flags.remote(), + } + + static args = { + modelResource: Args.string({description: 'The resource ID or alias of the model resource to check.'}), + } + + public async run(): Promise { + const {args, flags} = await this.parse(Info) + const {app} = flags + const {modelResource} = args + const synthesizedModels: Array = [] + let listOfProvisionedModels: Array = [] + + const modelInfo = async () => { + const modelInfoResponse = await this.herokuAI.get(`/models/${this.apiModelId}`, { + headers: {authorization: `Bearer ${this.apiKey}`}, + }) + .catch(error => { + if (error.statusCode === 404) { + ux.warn(`We can’t find a model resource called ${color.yellow(modelResource)}.\nRun ${color.cmd('heroku ai:models:info -a ')} to see a list of model resources.`) + } else { + throw error + } + }) + + return modelInfoResponse + } + + const getModelDetails = async (collectedModels: Array | string) => { + if (typeof collectedModels === 'string') { + const modelResource = collectedModels + await this.configureHerokuAIClient(modelResource, app) + + const {body: currentModelResource} = await modelInfo() || {body: null} + synthesizedModels.push(currentModelResource!) + } else { + for (const addonModel of collectedModels) { + await this.configureHerokuAIClient(addonModel.modelResource, app) + + const {body: currentModelResource} = await modelInfo() || {body: null} + synthesizedModels.push(currentModelResource!) + } + } + + return synthesizedModels + } + + if (modelResource) { + listOfProvisionedModels = await getModelDetails(modelResource) + } else { + const provisionedModelsInfo: Record[] = [] + const inferenceRegex = /inference/ + const addonsResponse = await appAddons(this.config, app) + + for (const addonInfo of addonsResponse as Array) { + const addonType = addonInfo.addon_service?.name || '' + const isModelAddon = inferenceRegex.test(addonType) + + if (isModelAddon) { + provisionedModelsInfo.push({ + addonName: addonInfo.addon_service?.name, + modelResource: addonInfo.name, + modelId: addonInfo.addon_service?.id, + }) + } + } + + listOfProvisionedModels = await getModelDetails(provisionedModelsInfo) + } + + this.displayModelResource(listOfProvisionedModels) + } + + displayModelResource(modelResources: ModelResource[]) { + for (const modelResource of modelResources) { + ux.log() + ux.styledHeader(modelResource.model_id) + ux.styledObject({ + 'Base Model ID': modelResource.model_id, + Ready: modelResource.ready, + 'Tokens In': modelResource.tokens_in, + 'Tokens Out': modelResource.tokens_out, + 'Avg Performance': modelResource.avg_performance, + }) + } + } +} diff --git a/src/lib/ai/models/app_addons.ts b/src/lib/ai/models/app_addons.ts new file mode 100644 index 0000000..bdad9ae --- /dev/null +++ b/src/lib/ai/models/app_addons.ts @@ -0,0 +1,17 @@ +import {Config} from '@oclif/core' +import {APIClient} from '@heroku-cli/command' +import * as Heroku from '@heroku-cli/schema' + +export default async function (config: Config, app: string) { + const herokuClient = new APIClient(config) + + const {body: response} = await herokuClient.get(`/apps/${app}/addons`, { + headers: {'Accept-Expansion': 'plan'}, + }).catch(error => { + console.log('ERROR MESSAGE:', error.message) + const error_ = error.body && error.body.message ? new Error(`The add-on was unable to be destroyed: ${error.body.message}.`) : new Error(`The add-on was unable to be destroyed: ${error}.`) + throw error_ + }) + + return response +} diff --git a/src/lib/ai/models/destroy_addon.ts b/src/lib/ai/models/destroy_addon.ts new file mode 100644 index 0000000..8ed468a --- /dev/null +++ b/src/lib/ai/models/destroy_addon.ts @@ -0,0 +1,23 @@ +import color from '@heroku-cli/color' +import {ux} from '@oclif/core' +import {Config} from '@oclif/core' +import {APIClient} from '@heroku-cli/command' +import * as Heroku from '@heroku-cli/schema' + +export default async function (config: Config, addon: Heroku.AddOn, force = false) { + const addonName = addon.name || '' + const herokuClient = new APIClient(config) + + ux.action.start(`Destroying ${color.addon(addonName)} in the background.\nThe app will restart when complete...`) + + await herokuClient.delete(`/apps/${addon.app?.id}/addons/${addon.id}`, { + headers: {'Accept-Expansion': 'plan'}, + body: {force}, + }).catch(error => { + ux.action.stop('') + const error_ = error.body && error.body.message ? new Error(`The add-on was unable to be destroyed: ${error.body.message}.`) : new Error(`The add-on was unable to be destroyed: ${error}.`) + throw error_ + }) + + ux.action.stop() +} diff --git a/src/lib/ai/types.ts b/src/lib/ai/types.ts index 4ec9a00..0ae4451 100644 --- a/src/lib/ai/types.ts +++ b/src/lib/ai/types.ts @@ -42,7 +42,8 @@ export type ModelInfo = { * Object schema for Model Status endpoint responses. */ export type ModelResource = { - plan: ModelName + model_id: ModelName + ready: string created: string tokens_in: string tokens_out?: string diff --git a/src/lib/base.ts b/src/lib/base.ts index 7868913..b376fa2 100644 --- a/src/lib/base.ts +++ b/src/lib/base.ts @@ -18,6 +18,18 @@ export class NotFound extends Error { public readonly id = 'not_found' } +export class AppNotFound extends Error { + constructor(appIdentifier?: string) { + const message = heredoc` + We can’t find the ${color.app(appIdentifier)} app. Check your spelling. + ` + super(message) + } + + public readonly statusCode = 404 + public readonly id = 'not_found' +} + export class AmbiguousError extends Error { constructor(public readonly matches: string[], addonIdentifier: string, appIdentifier?: string) { const message = heredoc` @@ -173,8 +185,9 @@ export default abstract class extends Command { } // 5. If we resolved for an add-on, check that it's a Managed Inference add-on or throw a NotFound error. - if (resolvedAddon && resolvedAddon.addon_service.name !== this.addonServiceSlug) + if (resolvedAddon && resolvedAddon.addon_service.name !== this.addonServiceSlug) { throw new NotFound(addonIdentifier, appIdentifier) + } // 6. If we resolved for an add-on but not for an attachment yet, try to resolve the attachment if (resolvedAddon && !resolvedAttachment) { @@ -210,11 +223,17 @@ export default abstract class extends Command { const attachmentNotFound = attachmentResolverError instanceof HerokuAPIError && attachmentResolverError.http.statusCode === 404 && attachmentResolverError.body.resource === 'add_on attachment' + const appNotFound = attachmentResolverError instanceof HerokuAPIError && + attachmentResolverError.http.statusCode === 404 && + attachmentResolverError.body.resource === 'app' let error = addonResolverError if (addonNotFound) error = attachmentNotFound ? new NotFound(addonIdentifier, appIdentifier) : attachmentResolverError + if (appNotFound) + error = new AppNotFound(appIdentifier) + throw error } diff --git a/test/commands/ai/models/destroy.test.ts b/test/commands/ai/models/destroy.test.ts new file mode 100644 index 0000000..a8edf63 --- /dev/null +++ b/test/commands/ai/models/destroy.test.ts @@ -0,0 +1,83 @@ +import {expect} from 'chai' +import {stderr, stdout} from 'stdout-stderr' +import Cmd from '../../../../src/commands/ai/models/destroy' +import stripAnsi from '../../../helpers/strip-ansi' +import {runCommand} from '../../../run-command' +import {mockConfigVars, mockAPIErrors, addon1, addon1Attachment1} from '../../../helpers/fixtures' +import {CLIError} from '@oclif/core/lib/errors' +import nock from 'nock' + +describe('ai:models:destroy', function () { + const {env} = process + let api: nock.Scope + + beforeEach(function () { + process.env = {} + api = nock('https://api.heroku.com:443') + }) + + afterEach(function () { + process.env = env + api.done() + nock.cleanAll() + }) + + context('no model resource is provided', function () { + it('errors when no model resource is provided', async function () { + try { + await runCommand(Cmd) + } catch (error) { + const {message} = error as CLIError + expect(stripAnsi(message)).contains('Missing 1 required arg:') + expect(stripAnsi(message)).contains('modelResource The resource ID or alias of the model resource to destroy.') + } + }) + }) + + it('displays confirmation of AI addon destruction', async function () { + const addonAppId = addon1.app?.id + const addonId = addon1.id + const addonName = addon1.name + const appName = addon1.app?.name + + api + .post('/actions/addons/resolve', {app: `${appName}`, addon: `${addonName}`}) + .reply(200, [addon1]) + .get(`/addons/${addonId}/addon-attachments`) + .reply(200, [addon1Attachment1]) + .get(`/apps/${addonAppId}/config-vars`) + .reply(200, mockConfigVars) + .delete(`/apps/${addonAppId}/addons/${addonId}`, {force: false}) + .reply(200, {...addon1, state: 'deprovisioned'}) + + await runCommand(Cmd, [`${addonName}`, '--app', `${appName}`, '--confirm', `${appName}`]) + expect(stderr.output).contains(`Destroying ${addonName} in the background.`) + expect(stderr.output).contains('The app will restart when complete...') + expect(stdout.output).to.eq('') + }) + + it('displays API error message if destroy request fails', async function () { + const addonAppId = addon1.app?.id + const addonId = addon1.id + const addonName = addon1.name + const appName = addon1.app?.name + + api + .post('/actions/addons/resolve', {app: `${appName}`, addon: `${addonName}`}) + .reply(200, [addon1]) + .get(`/addons/${addonId}/addon-attachments`) + .reply(200, [addon1Attachment1]) + .get(`/apps/${addonAppId}/config-vars`) + .reply(200, mockConfigVars) + .delete(`/apps/${addonAppId}/addons/${addonId}`, {force: false}) + .reply(500, mockAPIErrors.modelsDestroyErrorResponse) + + try { + await runCommand(Cmd, [`${addonName}`, '--app', `${appName}`, '--confirm', `${appName}`]) + } catch (error) { + const {message} = error as CLIError + expect(stripAnsi(message)).to.contains('The add-on was unable to be destroyed:') + expect(stripAnsi(message)).to.contains(mockAPIErrors.modelsDestroyErrorResponse.message) + } + }) +}) diff --git a/test/commands/ai/models/detach.test.ts b/test/commands/ai/models/detach.test.ts new file mode 100644 index 0000000..7f93245 --- /dev/null +++ b/test/commands/ai/models/detach.test.ts @@ -0,0 +1,47 @@ +import {stdout, stderr} from 'stdout-stderr' +import Cmd from '../../../../src/commands/ai/models/detach' +import {runCommand} from '../../../run-command' +import nock from 'nock' +import {expect} from 'chai' +import {addon1, addon1Attachment1, mockConfigVars} from '../../../helpers/fixtures' + +describe('addons:detach', function () { + let api: nock.Scope + const {env} = process + + beforeEach(function () { + process.env = {} + api = nock('https://api.heroku.com:443') + }) + + afterEach(function () { + process.env = env + api.done() + nock.cleanAll + }) + + it('detaches an add-on', async function () { + const addonAppId = addon1.app?.id + const addonId = addon1.id + const addonName = addon1.name + const appName = addon1.app?.name + + api + .post('/actions/addons/resolve', {app: `${appName}`, addon: `${addonName}`}) + .reply(200, [addon1]) + .get(`/addons/${addonId}/addon-attachments`) + .reply(200, [addon1Attachment1]) + .get(`/apps/${addonAppId}/config-vars`) + .reply(200, mockConfigVars) + .delete(`/addon-attachments/${addonId}`) + .reply(200) + .get(`/apps/${appName}/releases`) + .reply(200, [{version: 10}]) + + await runCommand(Cmd, [`${addonName}`, '--app', `${appName}`]) + + expect(stdout.output).to.equal('') + expect(stderr.output).to.contain(`Detaching ${addonName} from ${appName}... done`) + expect(stderr.output).to.contain(`Unsetting ${addonName} config vars and restarting ${appName}... done, v10`) + }) +}) diff --git a/test/commands/ai/models/info.test.ts b/test/commands/ai/models/info.test.ts new file mode 100644 index 0000000..fd9ab2a --- /dev/null +++ b/test/commands/ai/models/info.test.ts @@ -0,0 +1,171 @@ +import {expect} from 'chai' +import {stdout, stderr} from 'stdout-stderr' +import Cmd from '../../../../src/commands/ai/models/info' +import {runCommand} from '../../../run-command' +import {modelResource, addon1Attachment1, addon1, mockAPIErrors} from '../../../helpers/fixtures' +import nock from 'nock' +import heredoc from 'tsheredoc' +import stripAnsi from '../../../helpers/strip-ansi' +import {CLIError} from '@oclif/core/lib/errors' + +describe('ai:models:info', function () { + const {env} = process + let api: nock.Scope + let herokuAI: nock.Scope + + context('when provisioned model name is provided and is found', function () { + beforeEach(function () { + process.env = {} + api = nock('https://api.heroku.com:443') + herokuAI = nock('https://inference.heroku.com') + }) + + afterEach(function () { + process.env = env + nock.cleanAll() + }) + + it('shows info for a model resource', async function () { + api + .post('/actions/addons/resolve', + {addon: addon1.name, app: addon1Attachment1.app?.name}) + .reply(200, [addon1]) + .get(`/addons/${addon1.id}/addon-attachments`) + .reply(200, [addon1Attachment1]) + .get(`/apps/${addon1Attachment1.app?.id}/config-vars`) + .reply(200, { + INFERENCE_KEY: 's3cr3t_k3y', + INFERENCE_MODEL_ID: 'claude-3-haiku', + INFERENCE_URL: 'inference.heroku.com', + }) + herokuAI + .get('/models/claude-3-haiku') + .reply(200, modelResource) + + await runCommand(Cmd, [ + 'inference-regular-74659', + '--app', + 'app1', + ]) + + expect(stripAnsi(stdout.output)).to.equal(heredoc` + + === claude-3-haiku + + Avg Performance: latency 0.4sec, 28 tokens/sec + Base Model ID: claude-3-haiku + Ready: Yes + Tokens In: 0 tokens this period + Tokens Out: 0 tokens this period + `) + + expect(stderr.output).to.eq('') + }) + }) + + context('when provisioned model name is not provided', function () { + // eslint-disable-next-line mocha/no-setup-in-describe + const multipleAddons = Array.from({length: 2}).fill(addon1) + + beforeEach(function () { + process.env = {} + api = nock('https://api.heroku.com:443') + }) + + afterEach(function () { + process.env = env + nock.cleanAll() + }) + + it('shows info for all model resources on specified app', async function () { + api + .post('/actions/addons/resolve', + {addon: addon1.name, app: addon1Attachment1.app?.name}) + .reply(200, [addon1]) + .get(`/addons/${addon1.id}/addon-attachments`) + .reply(200, [addon1Attachment1]) + .get(`/apps/${addon1Attachment1.app?.id}/config-vars`) + .reply(200, { + INFERENCE_KEY: 's3cr3t_k3y', + INFERENCE_MODEL_ID: 'claude-3-haiku', + INFERENCE_URL: 'inference.heroku.com', + }) + herokuAI + .get('/models/claude-3-haiku') + .reply(200, modelResource) + api + .get(`/apps/${addon1.app?.name}/addons`) + .reply(200, multipleAddons) + .post('/actions/addons/resolve', + {addon: addon1.name, app: addon1Attachment1.app?.name}) + .reply(200, [addon1]) + .get(`/addons/${addon1.id}/addon-attachments`) + .reply(200, [addon1Attachment1]) + .get(`/apps/${addon1Attachment1.app?.id}/config-vars`) + .reply(200, { + INFERENCE_KEY: 's3cr3t_k3y', + INFERENCE_MODEL_ID: 'claude-3-haiku', + INFERENCE_URL: 'inference.heroku.com', + }) + herokuAI + .get('/models/claude-3-haiku') + .reply(200, modelResource) + + await runCommand(Cmd, [ + '--app', + 'app1', + ]) + + expect(stdout.output).to.equal(heredoc` + + === claude-3-haiku + + Avg Performance: latency 0.4sec, 28 tokens/sec + Base Model ID: claude-3-haiku + Ready: Yes + Tokens In: 0 tokens this period + Tokens Out: 0 tokens this period + + === claude-3-haiku + + Avg Performance: latency 0.4sec, 28 tokens/sec + Base Model ID: claude-3-haiku + Ready: Yes + Tokens In: 0 tokens this period + Tokens Out: 0 tokens this period + `) + }) + }) + + context('when provisioned model name is incorrectly inputted', function () { + const incorrectModelName = 'inference-regular-WRONG' + + beforeEach(function () { + process.env = {} + api = nock('https://api.heroku.com:443') + }) + + afterEach(function () { + process.env = env + nock.cleanAll() + }) + + it('shows an error message', async function () { + api + .post('/actions/addons/resolve', + {addon: incorrectModelName, app: addon1Attachment1.app?.name}) + .reply(404, mockAPIErrors.modelsInfoErrorResponse) + + try { + await runCommand(Cmd, [ + incorrectModelName, + '--app', + 'app1', + ]) + } catch (error) { + const {message} = error as CLIError + expect(stripAnsi(message)).contains(mockAPIErrors.modelsInfoErrorResponse.message) + } + }) + }) +}) diff --git a/test/helpers/fixtures.ts b/test/helpers/fixtures.ts index 5b9394a..ccd766c 100644 --- a/test/helpers/fixtures.ts +++ b/test/helpers/fixtures.ts @@ -1,5 +1,5 @@ import * as Heroku from '@heroku-cli/schema' -import {ChatCompletionResponse, EmbeddingResponse, ImageResponse} from '../../src/lib/ai/types' +import {ChatCompletionResponse, EmbeddingResponse, ImageResponse, ModelResource} from '../../src/lib/ai/types' export const availableModels = [ { @@ -32,11 +32,34 @@ export const availableModels = [ }, ] +export const mockConfigVars = { + INFERENCE_KEY: 's3cr3t_k3y', + INFERENCE_MODEL_ID: 'claude-3-opus', + INFERENCE_URL: 'inference-eu.heroku.com', +} + export const mockAPIErrors = { modelsListErrorResponse: { id: 'error', message: 'Failed to retrieve the list of available models. Check the Heroku Status page https://status.heroku.com/ for system outages. After all incidents have resolved, try again. You can also see a list of models at https://devcenter.heroku.com/articles/rainbow-unicorn-princess-models.', }, + modelsDestroyErrorResponse: { + id: 'error', + message: 'Example API Error', + }, + modelsInfoErrorResponse: { + id: 'error', + message: 'Example API Error', + }, +} + +export const modelResource: ModelResource = { + model_id: 'claude-3-haiku', + ready: 'Yes', + created: '2023-01-21T13:02:37.320+00.00', + tokens_in: '0 tokens this period', + tokens_out: '0 tokens this period', + avg_performance: 'latency 0.4sec, 28 tokens/sec', } export const addon1: Heroku.AddOn = { diff --git a/test/lib/base.test.ts b/test/lib/base.test.ts index aa401a4..7a2b0b8 100644 --- a/test/lib/base.test.ts +++ b/test/lib/base.test.ts @@ -188,6 +188,32 @@ describe('attempt a request using the Heroku AI client', function () { }) }) + context('when using an existent model resource name with non-existent app', function () { + beforeEach(async function () { + api + .post('/actions/addons/resolve', {addon: addon1.name, app: 'app2'}) + .reply(404, [addon1]) + .post('/actions/addon-attachments/resolve', {addon_attachment: addon1.name, app: 'app2'}) + .reply(404, {id: 'not_found', message: 'Couldn\'t find that app.', resource: 'app'}) + }) + + it('returns a custom not found error message', async function () { + try { + await runCommand(CommandConfiguredWithResourceName, [ + addon1.name as string, + '--app=app2', + ]) + } catch (error) { + const {message} = error as Error + expect(stripAnsi(message)).to.equal(heredoc` + We can’t find the app2 app. Check your spelling. + `) + } + + expect(stdout.output).to.equal('') + }) + }) + context('when using the add-on service slug and no app, matching multiple model resources', function () { beforeEach(async function () { api