Merge branch 'main' into typo-rerank-2

.github/scripts/check-mdx-frontmatter.js

@@ -0,0 +1,109 @@
+const fs = require("fs").promises;
+const path = require("path");
+const matter = require("gray-matter");
+
+const mdxDir = path.join(__dirname, "../../fern/pages");
+const filePattern = /\.mdx$/;
+
+// Counters
+let totalFilesChecked = 0;
+let totalFilesValid = 0;
+let totalFilesInvalid = 0;
+
+// List of folders to exclude (relative to mdxDir)
+const excludedFolders = ["-ARCHIVE-", "api-reference", "llm-university"];
+
+function shouldExcludeFolder(dirPath) {
+  return excludedFolders.some((excludedFolder) => {
+    return path.relative(mdxDir, dirPath).startsWith(excludedFolder);
+  });
+}
+
+async function shouldExcludeFile(filePath) {
+  try {
+    const fileContent = await fs.readFile(filePath, "utf8");
+    const { data } = matter(fileContent);
+    return data.hidden === true;
+  } catch (error) {
+    console.error(`Error reading file "${filePath}":`, error);
+    return false; // In case of error, don't exclude the file
+  }
+}
+
+async function checkDescriptionLength(filePath) {
+  totalFilesChecked++;
+  const fileContent = await fs.readFile(filePath, "utf8");
+  const { data } = matter(fileContent);
+
+  if (!data.description) {
+    console.log(`File "${filePath}" is missing a description.`);
+    totalFilesInvalid++;
+    return false;
+  }
+
+  const descriptionLength = data.description.length;
+
+  if (descriptionLength < 50 || descriptionLength > 160) {
+    console.log(
+      `File "${filePath}" has an invalid description length: ${descriptionLength} characters.`
+    );
+    totalFilesInvalid++;
+    return false;
+  }
+
+  totalFilesValid++;
+  return true;
+}
+
+async function checkMDXFiles(dirPath) {
+  let allFilesValid = true;
+  const files = await fs.readdir(dirPath);
+
+  for (const file of files) {
+    const fullPath = path.join(dirPath, file);
+    const stat = await fs.lstat(fullPath);
+
+    if (stat.isDirectory()) {
+      if (shouldExcludeFolder(fullPath)) {
+        console.log(`Skipping excluded directory: ${fullPath}`);
+        continue;
+      }
+      const isValid = await checkMDXFiles(fullPath);
+      if (!isValid) {
+        allFilesValid = false;
+      }
+    } else if (filePattern.test(file)) {
+      if (await shouldExcludeFile(fullPath)) {
+        console.log(`Skipping excluded file: ${fullPath}`);
+        continue;
+      }
+      const isValid = await checkDescriptionLength(fullPath);
+      if (!isValid) {
+        allFilesValid = false;
+      }
+    }
+  }
+
+  return allFilesValid;
+}
+
+(async () => {
+  const allFilesValid = await checkMDXFiles(mdxDir);
+
+  // Summary report
+  console.log(`\nSummary Report:`);
+  console.log(`Total files checked: ${totalFilesChecked}`);
+  console.log(`Total valid files: ${totalFilesValid}`);
+  console.log(`Total invalid files: ${totalFilesInvalid}`);
+
+  if (!allFilesValid) {
+    console.error(
+      "Some files have invalid or missing descriptions. Meta description needing to be 50-160 characters"
+    );
+    process.exit(1); // Fail if any file is invalid
+  } else {
+    console.log(
+      "All files have a valid description length in the frontmatter."
+    );
+  }
+})();

.github/workflows/check-mdx-frontmatter.yml

@@ -0,0 +1,29 @@
+name: check-mdx-frontmatter
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'fern/pages/**/*.mdx'
+      - 'fern/pages/**/**/*.mdx'
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    permissions: write-all
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v2
+        with:
+          version: 8
+
+      - name: Install Dependencies
+        shell: bash
+        run: pnpm install
+
+      - name: Run MDX frontmatter check
+        run: node .github/scripts/check-mdx-frontmatter.js

fern/assets/input.css

@@ -1,6 +1,10 @@
 @tailwind components;
 @tailwind utilities;
 
+.fern-changelog>main {
+    width: 100%;
+}
+
 h1,
 h2,
 h3,
@@ -630,3 +634,10 @@ button[class^="Sidebar-link-buttonWrapper"] {
 .extra-green {
   color: #38761d !important;
 }
+
+/*
+    This is hacky fix for the code block comment color
+*/
+.code-block-line-content span[style*="color: rgb(194, 195, 197)"] {
+    color: rgb(155, 156, 158) !important;
+}

fern/docs.yml

@@ -1,3 +1,6 @@
+announcement:
+    message: "🚀 Announcement: New API V2 endpoints are available! (<a href=\"https://cohere.com/blog/new-api-v2\" target=\"_blank\">Learn more</a>) 🚀"
+
 instances:
   - url: cohere.docs.buildwithfern.com
     custom-domain: docs.cohere.com
@@ -11,10 +14,14 @@ title: Cohere
 default-language: python
 
 versions:
-  - display-name: v1
+  - display-name: v2 API
+    path: v2.yml
+    slug: v2
+  - display-name: v1 API
     path: v1.yml
     slug: v1
 
+
 logo:
   light: assets/logo.svg
   dark: assets/logo-dark.svg
@@ -106,6 +113,27 @@ redirects:
   - source: "/docs/llmu{/:path}*"
     destination: "/docs/llmu-2"
     permanent: true
+  - source: "/docs/managing-your-connector"
+    destination: "/v1/docs/managing-your-connector"
+  - source: "/docs/connector-authentication"
+    destination: "/v1/docs/connector-authentication"
+  - source: "/docs/connector-faqs"
+    destination: "/v1/docs/connector-faqs"
+  - source: "/docs/overview-rag-connectors"
+    destination: "/v1/docs/overview-rag-connectors"
+  - source: "/docs/migrating-from-cogenerate-to-cochat"
+    destination: "/v1/docs/migrating-from-cogenerate-to-cochat"
+  - source: "/reference/generate"
+    destination: "/v1/reference/generate"
+  - source: "/v2/docs/embed-2"
+    destination: "/docs/cohere-embed"
+  - source: "/docs/docs/overview-rag-connectors"
+    destination: "/v1/docs/overview-rag-connectors"
+  - source: "/docs/prompt-truncation"
+    destination: "/v1/docs/prompt-truncation"
+  - source: "/classify-reference"
+    destination: "/reference/classify"
+
   - source: "/docs/llmu"
     destination: "/docs/llmu-2"
     permanent: true

fern/fern.config.json

@@ -1,4 +1,4 @@
 {
   "organization": "cohere",
   "version": "*"
-}
+}

fern/pages/api-reference/connectors-1/create-connector.mdx

@@ -1,7 +1,7 @@
 ---
 title: "Create a Connector"
 slug: "create-connector"
-subtitle: "Creates a new connector. The connector is tested during registration and will cancel registration when the test is unsuccessful. See ['Creating and Deploying a Connector'](/docs/creating-and-deploying-a-connector) for more information."
+subtitle: "Creates a new connector. The connector is tested during registration and will cancel registration when the test is unsuccessful. See ['Creating and Deploying a Connector'](/v1/docs/creating-and-deploying-a-connector) for more information."
 hidden: false
 createdAt: "Mon Jun 24 2024 13:21:35 GMT+0000 (Coordinated Universal Time)"
 updatedAt: "Mon Jun 24 2024 13:21:37 GMT+0000 (Coordinated Universal Time)"

fern/pages/changelog/2022-12-12-multilingual-text-understanding-model-language-detection.mdx

@@ -8,6 +8,8 @@ description: >-
   Cohere's multilingual model now supports semantic search across 100 languages
   with a single index.
 ---
+**[NOTE: `co.detect_language()` is deprecated.]**
+
 Cohere's multilingual text understanding model is now available! The `multilingual-22-12` model can be used to semantically search within a single language, as well as across languages. Compared to keyword search, where you often need separate tokenizers and indices to handle different languages, the deployment of the multilingual model for search is trivial: no language-specific handling is needed — everything can be done by a single model within a single index.
 
 

fern/pages/changelog/2023-06-28-release-notes-june-28th-2023.mdx

@@ -18,4 +18,4 @@ The Command model has been updated. Expect improved code and conversational capa
 Token streaming is now supported via the Co.generate() api. [Learn more here](/reference/generate).
 
 **Usage and Billing Table Improvements**  
-The usage and billing table in the Cohere dashboard now has filtering and sorting capabilities. [See here.](https://dashboard.cohere.ai/billing)
+The usage and billing table in the Cohere dashboard now has filtering and sorting capabilities. [See here.](https://dashboard.cohere.com/welcome/login?redirect_uri=%2Fbilling)

fern/pages/changelog/2024-01-02-release-notes-january-x-2024.mdx

@@ -14,7 +14,7 @@ One of the most exciting applications of generative AI is known as ["retrieval a
 
 Cohere has made it much easier to utilize RAG in bespoke applications via [Connectors](/docs/overview-rag-connectors). As the name implies, Connectors allow you to _connect_ Cohere's generative AI platform up to whatever resources you'd like it to ground on, facilitating the creation of a wide variety of applications -- customer service chatbots, internal tutors, or whatever else you want to build.
 
-Our docs cover how to [create and deploy connectors](/docs/creating-and-deploying-a-connector), [how to manage your connectors ](/docs/managing-your-connector), [how to handle authentication](/docs/connector-authentication), and [more](/docs/connector-faqs)!
+Our docs cover how to [create and deploy connectors](/v1/docs/creating-and-deploying-a-connector), [how to manage your connectors ](/docs/managing-your-connector), [how to handle authentication](/docs/connector-authentication), and [more](/docs/connector-faqs)!
 
 ## Expanded Fine-tuning Functionality
 

fern/pages/changelog/2024-09-26-api-v2.mdx

@@ -0,0 +1,35 @@
+---
+title: "New Embed, Rerank, Chat, and Classify APIs"
+slug: "changelog/v2-api-release"
+createdAt: "Thurs Sept 26 2024 09:30:00 (EST)"
+hidden: false
+description: >-
+  Introducing improvements to our Chat, Classify, Embed, and Rerank APIs in a major version upgrade, making it easier and faster to build with Cohere.
+---
+We're excited to introduce improvements to our Chat, Classify, Embed, and Rerank APIs in a major version upgrade, making it easier and faster to build with Cohere. We are also releasing new versions of our Python, TypeScript, Java, and Go SDKs which feature `cohere.ClientV2` for access to the new API. 
+
+## New at a glance
+* V2 Chat, Classify, Embed, and Rerank: `model` is a required parameter
+* V2 Embed: `embedding_types` is a required parameter
+* V2 Chat: Message and chat history are [combined](/v2/docs/migrating-v1-to-v2#messages-and-preamble) in a single `messages` array
+* V2 Chat: [Tools](/v2/docs/parameter-types-in-tool-use) are defined in JSON schema
+* V2 Chat: Introduces `tool_call_ids` to [match tool calls with tool results](/v2/docs/migrating-v1-to-v2#tool-call-id)
+* V2 Chat: `documents` [supports a list of strings or a list of objects](/v2/docs/migrating-v1-to-v2#documents) with document metadata
+* V2 Chat streaming: Uses [server-sent events](/v2/docs/migrating-v1-to-v2#streaming)
+
+## Other updates
+We are simplifying the Chat API by removing support for the following parameters available in V1:
+* `search_queries_only`, which generates only a search query given a user’s message input. `search_queries_only` is not supported in the V2 Chat API today, but will be supported at a later date. 
+* `connectors`, which enables users to register a data source with Cohere for RAG queries. To use the Chat V2 API with web search, see our [migration guide](/v2/docs/migrating-v1-to-v2#) for instructios to implement a web search tool.
+* `conversation_id`, used to manage chat history on behalf of the developer. This will not be supported in the V2 Chat API.
+* `prompt_truncation`, used to automatically rerank and remove documents if the query did not fit in the model’s context limit. This will not be supported in the V2 Chat API.
+* `force_single_step`, which forced the model to finish tool calling in one set of turns. This will not be supported in the V2 Chat API.
+* `preamble`, used for giving the model task, context, and style instructions. Use a system turn at the beginning of your `messages` array in V2.
+* `citation_quality`, for users to select between `fast` citations, `accurate` citations (slightly higher latency than fast), or citations `off`. In V2 Chat, we are introducing a top level `citation_options` parameter for all citation settings. `citation_quality` will be replaced by a `mode` parameter within `citation_options`.
+
+
+See our Chat API [migration guide](/v2/docs/migrating-v1-to-v2) for detailed instructions to update your implementation.
+
+
+These APIs are in Beta and are subject to updates. We welcome feedback in our [Discord](https://discord.com/invite/co-mmunity) channel.
+

fern/pages/changelog/2024-09-26-refresh-models-on-azure.mdx

@@ -0,0 +1,14 @@
+---
+title: "The refreshed Command R and Command R+ models are now on Azure"
+slug: "changelog/refresh-models-on-azure"
+createdAt: "Thurs Sept 24 2024"
+hidden: false
+description: >-
+  Introducing our improved Command models are available on the Azure cloud computing platform.
+---
+
+You'll recall that we [released refreshed models](https://docs.cohere.com/changelog/command-gets-refreshed) of Command R and Command R+ in August. 
+
+Today, we're pleased to announce that these models are available on the Azure cloud computing platform!
+
+You can find more information about using Cohere's Command models on Azure [here](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/deploy-models-cohere-command?tabs=cohere-command-r-plus&pivots=programming-language-python).

fern/pages/changelog/2024-10-03-commandr-082024-ft.mdx

@@ -0,0 +1,16 @@
+---
+title: "Fine-Tuning Now Available for Command R 08-2024"
+slug: "changelog/commandr-082024-ft"
+createdAt: "Thurs Oct 03 2024"
+hidden: false
+description: >-
+  Launch of fine-tuning for Command R 08-2024 and other new fine-tuning features.
+---
+
+Today, we're pleased to announce that fine-tuning is now available for [Command R 08-2024](/docs/command-r#command-r-august-2024-release)!
+
+We're also introducing other chat fine-tuning features:
+- Support for **16384 context lengths (up from 8192)** for fine-tuning training and MultiLoRA.
+- Integration with Weights & Biases for tracking fine-tuning experiments in real time.
+
+See the [Chat fine-tuning documentation](/docs/chat-fine-tuning) to learn more about creating a fine-tuned model.

fern/pages/changelog/2024-10-22-Embed-v3-is-multimodal.mdx

@@ -0,0 +1,36 @@
+---
+title: "Embed v3.0 Models are now Multimodal"
+slug: "changelog/embed-v3-is-multimodal"
+createdAt: "Tues Oct 22 2024 05:30:00 (MST)"
+hidden: false
+description: >-
+  Launch of multimodal embeddings for our Embed models, plus some code to help get started.
+---
+
+Today we’re announcing updates to our embed-v3.0 family of models. These models now have the ability to process images into embeddings. There is no change to existing text capabilities which means there is no need to re-embed texts you have already processed with our `embed-v3.0` models.
+
+In the rest of these release notes, we’ll provide more details about technical enhancements, new features, and new pricing.
+
+## Technical Details
+### API Changes:
+The Embed API has two major changes: 
+- Introduced a new `input_type` called `image`
+- Introduced a new parameter called `images`
+
+Example request on how to process 
+
+```Text cURL
+POST https://api.cohere.ai/v1/embed
+{
+    "model": "embed-multilingual-v3.0",
+    "input_type": "image",
+    "embedding_types": ["float"],
+    "images": [enc_img]
+}
+```
+### Restrictions: 
+- The API only accepts images in the base format of the following: `png`, `jpeg`,`Webp`, and `gif`
+- Image embeddings currently does not support batching so the max images sent per request is 1
+- The maximum image sizez is `5mb`
+- The `images` parameter only accepts a base64 encoded image formatted as a Data Url
+