Skip to content

Commit

Permalink
feat(clients): add optionnal scopes to replaceAllObjects [skip-bc] (#…
Browse files Browse the repository at this point in the history
  • Loading branch information
millotp authored Jan 7, 2025
1 parent 0a370d1 commit e7b3898
Show file tree
Hide file tree
Showing 18 changed files with 370 additions and 57 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -140,11 +140,12 @@ public partial interface ISearchClient
/// <param name="indexName">The index in which to perform the request.</param>
/// <param name="objects">The list of `objects` to store in the given Algolia `indexName`.</param>
/// <param name="batchSize">The size of the chunk of `objects`. The number of `batch` calls will be equal to `length(objects) / batchSize`. Defaults to 1000.</param>
/// <param name="scopes"> The `scopes` to keep from the index. Defaults to ['settings', 'rules', 'synonyms'].</param>
/// <param name="options">Add extra http header or query parameters to Algolia.</param>
/// <param name="cancellationToken">Cancellation Token to cancel the request.</param>
Task<ReplaceAllObjectsResponse> ReplaceAllObjectsAsync<T>(string indexName, IEnumerable<T> objects, int batchSize = 1000, RequestOptions options = null, CancellationToken cancellationToken = default) where T : class;
Task<ReplaceAllObjectsResponse> ReplaceAllObjectsAsync<T>(string indexName, IEnumerable<T> objects, int batchSize = 1000, List<ScopeType> scopes = null, RequestOptions options = null, CancellationToken cancellationToken = default) where T : class;
/// <inheritdoc cref="ReplaceAllObjectsAsync{T}(string, IEnumerable{T}, int, RequestOptions, CancellationToken)"/>
ReplaceAllObjectsResponse ReplaceAllObjects<T>(string indexName, IEnumerable<T> objects, int batchSize = 1000, RequestOptions options = null, CancellationToken cancellationToken = default) where T : class;
ReplaceAllObjectsResponse ReplaceAllObjects<T>(string indexName, IEnumerable<T> objects, int batchSize = 1000, List<ScopeType> scopes = null, RequestOptions options = null, CancellationToken cancellationToken = default) where T : class;

/// <summary>
/// Helper: Chunks the given `objects` list in subset of 1000 elements max in order to make it fit in `batch` requests.
Expand Down Expand Up @@ -484,21 +485,26 @@ private static int NextDelay(int retryCount)

/// <inheritdoc/>
public async Task<ReplaceAllObjectsResponse> ReplaceAllObjectsAsync<T>(string indexName, IEnumerable<T> objects,
int batchSize = 1000, RequestOptions options = null, CancellationToken cancellationToken = default) where T : class
int batchSize = 1000, List<ScopeType> scopes = null, RequestOptions options = null, CancellationToken cancellationToken = default) where T : class
{
if (objects == null)
{
throw new ArgumentNullException(nameof(objects));
}

if (scopes == null)
{
scopes = new List<ScopeType> { ScopeType.Settings, ScopeType.Rules, ScopeType.Synonyms };
}

var rnd = new Random();
var tmpIndexName = $"{indexName}_tmp_{rnd.Next(100)}";

try
{
var copyResponse = await OperationIndexAsync(indexName,
new OperationIndexParams(OperationType.Copy, tmpIndexName)
{ Scope = [ScopeType.Settings, ScopeType.Rules, ScopeType.Synonyms] }, options, cancellationToken)
{ Scope = scopes }, options, cancellationToken)
.ConfigureAwait(false);

var batchResponse = await ChunkedBatchAsync(tmpIndexName, objects, Action.AddObject, true, batchSize,
Expand All @@ -509,7 +515,7 @@ await WaitForTaskAsync(tmpIndexName, copyResponse.TaskID, requestOptions: option

copyResponse = await OperationIndexAsync(indexName,
new OperationIndexParams(OperationType.Copy, tmpIndexName)
{ Scope = [ScopeType.Settings, ScopeType.Rules, ScopeType.Synonyms] }, options, cancellationToken)
{ Scope = scopes }, options, cancellationToken)
.ConfigureAwait(false);
await WaitForTaskAsync(tmpIndexName, copyResponse.TaskID, requestOptions: options, ct: cancellationToken)
.ConfigureAwait(false);
Expand Down Expand Up @@ -537,9 +543,9 @@ await WaitForTaskAsync(tmpIndexName, moveResponse.TaskID, requestOptions: option
}

/// <inheritdoc/>
public ReplaceAllObjectsResponse ReplaceAllObjects<T>(string indexName, IEnumerable<T> objects, int batchSize = 1000,
public ReplaceAllObjectsResponse ReplaceAllObjects<T>(string indexName, IEnumerable<T> objects, int batchSize = 1000, List<ScopeType> scopes = null,
RequestOptions options = null, CancellationToken cancellationToken = default) where T : class =>
AsyncHelper.RunSync(() => ReplaceAllObjectsAsync(indexName, objects, batchSize, options, cancellationToken));
AsyncHelper.RunSync(() => ReplaceAllObjectsAsync(indexName, objects, batchSize, scopes, options, cancellationToken));

/// <inheritdoc/>
public async Task<List<BatchResponse>> ChunkedBatchAsync<T>(string indexName, IEnumerable<T> objects,
Expand Down
1 change: 1 addition & 0 deletions clients/algoliasearch-client-go/.golangci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ linters:

# Deprecated
- execinquery
- exportloopref

issues:
exclude-generated: disable
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -462,12 +462,14 @@ public suspend fun SearchClient.partialUpdateObjects(
* @param indexName The index in which to perform the request.
* @param objects The list of objects to replace.
* @param batchSize The size of the batch. Default is 1000.
* @param scopes The `scopes` to keep from the index. Defaults to ['settings', 'rules', 'synonyms'].
* @return responses from the three-step operations: copy, batch, move.
*/
public suspend fun SearchClient.replaceAllObjects(
indexName: String,
objects: List<JsonObject>,
batchSize: Int = 1000,
scopes: List<ScopeType> = listOf(ScopeType.Settings, ScopeType.Rules, ScopeType.Synonyms),
requestOptions: RequestOptions? = null,
): ReplaceAllObjectsResponse {
val tmpIndexName = "${indexName}_tmp_${Random.nextInt(from = 0, until = 100)}"
Expand All @@ -478,7 +480,7 @@ public suspend fun SearchClient.replaceAllObjects(
operationIndexParams = OperationIndexParams(
operation = OperationType.Copy,
destination = tmpIndexName,
scope = listOf(ScopeType.Settings, ScopeType.Rules, ScopeType.Synonyms),
scope = scopes,
),
requestOptions = requestOptions,
)
Expand All @@ -499,7 +501,7 @@ public suspend fun SearchClient.replaceAllObjects(
operationIndexParams = OperationIndexParams(
operation = OperationType.Copy,
destination = tmpIndexName,
scope = listOf(ScopeType.Settings, ScopeType.Rules, ScopeType.Synonyms),
scope = scopes,
),
requestOptions = requestOptions,
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -353,6 +353,8 @@ package object extension {
* The list of objects to replace.
* @param batchSize
* The size of the batch. Default is 1000.
* @param scopes
* The `scopes` to keep from the index. Defaults to ['settings', 'rules', 'synonyms'].
* @param requestOptions
* Additional request configuration.
* @return
Expand All @@ -362,6 +364,7 @@ package object extension {
indexName: String,
objects: Seq[Any],
batchSize: Int = 1000,
scopes: Option[Seq[ScopeType]] = Some(Seq(ScopeType.Settings, ScopeType.Rules, ScopeType.Synonyms)),
requestOptions: Option[RequestOptions] = None
)(implicit ec: ExecutionContext): Future[ReplaceAllObjectsResponse] = {
val tmpIndexName = s"${indexName}_tmp_${scala.util.Random.nextInt(100)}"
Expand All @@ -373,7 +376,7 @@ package object extension {
operationIndexParams = OperationIndexParams(
operation = OperationType.Copy,
destination = tmpIndexName,
scope = Some(Seq(ScopeType.Settings, ScopeType.Rules, ScopeType.Synonyms))
scope = scopes
),
requestOptions = requestOptions
)
Expand All @@ -394,7 +397,7 @@ package object extension {
operationIndexParams = OperationIndexParams(
operation = OperationType.Copy,
destination = tmpIndexName,
scope = Some(Seq(ScopeType.Settings, ScopeType.Rules, ScopeType.Synonyms))
scope = scopes
),
requestOptions = requestOptions
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -548,13 +548,15 @@ public extension SearchClient {
/// - parameter indexName: The name of the index where to replace the objects
/// - parameter objects: The new objects
/// - parameter batchSize: The maximum number of objects to include in a batch
/// - parameter scopes: The `scopes` to keep from the index. Defaults to ['settings', 'rules', 'synonyms']
/// - parameter requestOptions: The request options
/// - returns: ReplaceAllObjectsResponse
@discardableResult
func replaceAllObjects(
indexName: String,
objects: [some Encodable],
batchSize: Int = 1000,
scopes: [ScopeType] = [.settings, .rules, .synonyms],
requestOptions: RequestOptions? = nil
) async throws -> ReplaceAllObjectsResponse {
let tmpIndexName = "\(indexName)_tmp_\(Int.random(in: 1_000_000 ..< 10_000_000))"
Expand All @@ -565,7 +567,7 @@ public extension SearchClient {
operationIndexParams: OperationIndexParams(
operation: .copy,
destination: tmpIndexName,
scope: [.settings, .rules, .synonyms]
scope: scopes
),
requestOptions: requestOptions
)
Expand All @@ -584,7 +586,7 @@ public extension SearchClient {
operationIndexParams: OperationIndexParams(
operation: .copy,
destination: tmpIndexName,
scope: [.settings, .rules, .synonyms]
scope: scopes
),
requestOptions: requestOptions
)
Expand Down
2 changes: 2 additions & 0 deletions scripts/cts/runCts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import { assertChunkWrapperValid } from './testServer/chunkWrapper.js';
import { startTestServer } from './testServer/index.js';
import { assertValidReplaceAllObjects } from './testServer/replaceAllObjects.js';
import { assertValidReplaceAllObjectsFailed } from './testServer/replaceAllObjectsFailed.js';
import { assertValidReplaceAllObjectsScopes } from './testServer/replaceAllObjectsScopes.js';
import { assertValidTimeouts } from './testServer/timeout.js';
import { assertValidWaitForApiKey } from './testServer/waitFor.js';

Expand Down Expand Up @@ -151,6 +152,7 @@ export async function runCts(
assertChunkWrapperValid(languages.length - skip('dart') - skip('scala'));
assertValidReplaceAllObjects(languages.length - skip('dart') - skip('scala'));
assertValidReplaceAllObjectsFailed(languages.length - skip('dart') - skip('scala'));
assertValidReplaceAllObjectsScopes(languages.length - skip('dart') - skip('scala'));
assertValidWaitForApiKey(languages.length - skip('dart') - skip('scala'));
}
if (withBenchmarkServer) {
Expand Down
2 changes: 2 additions & 0 deletions scripts/cts/testServer/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import { chunkWrapperServer } from './chunkWrapper.js';
import { gzipServer } from './gzip.js';
import { replaceAllObjectsServer } from './replaceAllObjects.js';
import { replaceAllObjectsServerFailed } from './replaceAllObjectsFailed.js';
import { replaceAllObjectsScopesServer } from './replaceAllObjectsScopes.js';
import { timeoutServer } from './timeout.js';
import { timeoutServerBis } from './timeoutBis.js';
import { waitForApiKeyServer } from './waitFor.js';
Expand All @@ -26,6 +27,7 @@ export async function startTestServer(suites: Record<CTSType, boolean>): Promise
timeoutServerBis(),
replaceAllObjectsServer(),
replaceAllObjectsServerFailed(),
replaceAllObjectsScopesServer(),
chunkWrapperServer(),
waitForApiKeyServer(),
apiKeyServer(),
Expand Down
122 changes: 122 additions & 0 deletions scripts/cts/testServer/replaceAllObjectsScopes.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
import type { Server } from 'http';

import { expect } from 'chai';
import type { Express } from 'express';
import express from 'express';

import { setupServer } from './index.js';

const raoState: Record<
string,
{
copyCount: number;
batchCount: number;
waitTaskCount: number;
tmpIndexName: string;
waitingForFinalWaitTask: boolean;
successful: boolean;
}
> = {};

export function assertValidReplaceAllObjectsScopes(expectedCount: number): void {
expect(Object.keys(raoState)).to.have.length(expectedCount);
for (const lang in raoState) {
expect(raoState[lang].successful).to.equal(true);
}
}

function addRoutes(app: Express): void {
app.use(express.urlencoded({ extended: true }));
app.use(
express.json({
type: ['application/json', 'text/plain'], // the js client sends the body as text/plain
}),
);

app.post('/1/indexes/:indexName/operation', (req, res) => {
expect(req.params.indexName).to.match(/^cts_e2e_replace_all_objects_scopes_(.*)$/);

switch (req.body.operation) {
case 'copy': {
expect(req.params.indexName).to.not.include('tmp');
expect(req.body.destination).to.include('tmp');
expect(req.body.scope).to.deep.equal(['settings', 'synonyms']);

const lang = req.params.indexName.replace('cts_e2e_replace_all_objects_scopes_', '');
if (!raoState[lang] || raoState[lang].successful) {
raoState[lang] = {
copyCount: 1,
batchCount: 0,
waitTaskCount: 0,
tmpIndexName: req.body.destination,
waitingForFinalWaitTask: false,
successful: false,
};
} else {
raoState[lang].copyCount++;
}

res.json({ taskID: 123 + raoState[lang].copyCount, updatedAt: '2021-01-01T00:00:00.000Z' });
break;
}
case 'move': {
const lang = req.body.destination.replace('cts_e2e_replace_all_objects_scopes_', '');
expect(raoState).to.include.keys(lang);
expect(raoState[lang]).to.deep.equal({
copyCount: 2,
batchCount: 2,
waitTaskCount: 3,
tmpIndexName: req.params.indexName,
waitingForFinalWaitTask: false,
successful: false,
});

expect(req.body.scope).to.equal(undefined);

raoState[lang].waitingForFinalWaitTask = true;

res.json({ taskID: 777, updatedAt: '2021-01-01T00:00:00.000Z' });

break;
}
default:
res.status(400).json({
message: `invalid operation: ${req.body.operation}, body: ${JSON.stringify(req.body)}`,
});
}
});

app.post('/1/indexes/:indexName/batch', (req, res) => {
const lang = req.params.indexName.match(/^cts_e2e_replace_all_objects_scopes_(.*)_tmp_\d+$/)?.[1] as string;
expect(raoState).to.include.keys(lang);
expect(req.body.requests.every((r) => r.action === 'addObject')).to.equal(true);

raoState[lang].batchCount += req.body.requests.length;

res.json({
taskID: 124 + raoState[lang].batchCount,
objectIDs: req.body.requests.map((r) => r.body.objectID),
});
});

app.get('/1/indexes/:indexName/task/:taskID', (req, res) => {
const lang = req.params.indexName.match(/^cts_e2e_replace_all_objects_scopes_(.*)_tmp_\d+$/)?.[1] as string;
expect(raoState).to.include.keys(lang);

raoState[lang].waitTaskCount++;
if (raoState[lang].waitingForFinalWaitTask) {
expect(req.params.taskID).to.equal('777');
expect(raoState[lang].waitTaskCount).to.equal(4);

raoState[lang].successful = true;
}

res.json({ status: 'published', updatedAt: '2021-01-01T00:00:00.000Z' });
});
}

export function replaceAllObjectsScopesServer(): Promise<Server> {
// this server is used to simulate the responses for the replaceAllObjects method with partial scopes,
// and uses a state machine to determine if the logic is correct.
return setupServer('replaceAllObjectsScopes', 6685, addRoutes);
}
8 changes: 8 additions & 0 deletions specs/search/helpers/replaceAllObjects.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,14 @@ method:
required: false
schema:
type: integer
- in: query
name: scopes
description: List of scopes to kepp in the index. Defaults to `settings`, `synonyms`, and `rules`.
required: false
schema:
type: array
items:
$ref: '../common/enums.yml#/scopeType'
responses:
'200':
description: OK
Expand Down
Loading

0 comments on commit e7b3898

Please sign in to comment.