feat: improve memory usage of record export

src/record/delete/parsers/error.ts

@@ -1,18 +1,11 @@
 import { CsvError } from "csv-parse";
+import { CliKintoneError } from "../../../utils/error";
 
-export class ParserError extends Error {
-  private readonly cause: unknown;
-
+export class ParserError extends CliKintoneError {
   constructor(cause: unknown) {
     const message = "Failed to parse input";
-    super(message);
-
+    super(message, cause);
     this.name = "ParserError";
-    this.message = message;
-    this.cause = cause;
-
-    // https://github.com/Microsoft/TypeScript/wiki/Breaking-Changes#extending-built-ins-like-error-array-and-map-may-no-longer-work
-    // Set the prototype explicitly.
     Object.setPrototypeOf(this, ParserError.prototype);
   }
 

src/record/delete/usecases/deleteAll/error.ts

@@ -2,20 +2,18 @@ import { KintoneAllRecordsError } from "@kintone/rest-api-client";
 import type { KintoneRecordForDeleteAllParameter } from "../../../../kintone/types";
 import { kintoneAllRecordsErrorToString } from "../../../error";
 import { ErrorParser } from "../../utils/error";
+import { CliKintoneError } from "../../../../utils/error";
 
-export class DeleteAllRecordsError extends Error {
-  private readonly cause: unknown;
+export class DeleteAllRecordsError extends CliKintoneError {
   private readonly records: KintoneRecordForDeleteAllParameter[];
   private readonly numOfSuccess: number;
   private readonly numOfTotal: number;
 
   constructor(cause: unknown, records: KintoneRecordForDeleteAllParameter[]) {
     const message = "Failed to delete all records.";
-    super(message);
+    super(message, cause);
 
     this.name = "DeleteAllRecordsError";
-    this.message = message;
-    this.cause = cause;
     this.records = records;
     this.numOfTotal = this.records.length;
     this.numOfSuccess = 0;

src/record/delete/usecases/deleteByRecordNumber/error.ts

@@ -2,20 +2,18 @@ import { KintoneAllRecordsError } from "@kintone/rest-api-client";
 import type { KintoneRecordForDeleteAllParameter } from "../../../../kintone/types";
 import { kintoneAllRecordsErrorToString } from "../../../error";
 import { ErrorParser } from "../../utils/error";
+import { CliKintoneError } from "../../../../utils/error";
 
-export class DeleteSpecifiedRecordsError extends Error {
-  private readonly cause: unknown;
+export class DeleteSpecifiedRecordsError extends CliKintoneError {
   private readonly records: KintoneRecordForDeleteAllParameter[];
   private readonly numOfSuccess: number;
   private readonly numOfTotal: number;
 
   constructor(cause: unknown, records: KintoneRecordForDeleteAllParameter[]) {
     const message = "Failed to delete records.";
-    super(message);
+    super(message, cause);
 
     this.name = "DeleteSpecifiedRecordsError";
-    this.message = message;
-    this.cause = cause;
     this.records = records;
     this.numOfTotal = this.records.length;
     this.numOfSuccess = 0;

src/record/delete/validator/error.ts

@@ -1,13 +1,11 @@
-export class ValidatorError extends Error {
-  private readonly cause: unknown;
+import { CliKintoneError } from "../../../utils/error";
 
+export class ValidatorError extends CliKintoneError {
   constructor(cause: unknown) {
     const message = "Failed to delete records";
-    super(message);
+    super(message, cause);
 
     this.name = "ValidatorError";
-    this.message = message;
-    this.cause = cause;
 
     // https://github.com/Microsoft/TypeScript/wiki/Breaking-Changes#extending-built-ins-like-error-array-and-map-may-no-longer-work
     // Set the prototype explicitly.

src/record/export/index.ts

@@ -3,11 +3,13 @@ import iconv from "iconv-lite";
 import type { RestAPIClientOptions } from "../../kintone/client";
 import { buildRestAPIClient } from "../../kintone/client";
 import { getRecords } from "./usecases/get";
-import { stringifierFactory } from "./stringifiers";
 import { createSchema } from "./schema";
 import { formLayout as defaultTransformer } from "./schema/transformers/formLayout";
 import { userSelected } from "./schema/transformers/userSelected";
 import { logger } from "../../utils/log";
+import { LocalRecordRepositoryFromStream } from "./repositories/localRecordRepositoryFromStream";
+import { ReadableStream } from "stream/web";
+import { Duplex, Readable } from "stream";
 
 export type ExportFileEncoding = "utf8" | "sjis";
 
@@ -42,18 +44,22 @@ export const run: (
         ? userSelected(fields, fieldsJson, layoutJson)
         : defaultTransformer(layoutJson)
     );
-    const records = await getRecords(apiClient, app, schema, {
+
+    const repository = new LocalRecordRepositoryFromStream(
+      () => {
+        const stream = iconv.encodeStream(encoding);
+        stream.pipe(process.stdout);
+        return stream;
+      },
+      schema,
+      !!attachmentsDir
+    );
+
+    await getRecords(apiClient, app, repository, schema, {
       condition,
       orderBy,
       attachmentsDir,
     });
-    const stringifier = stringifierFactory({
-      format: "csv",
-      schema,
-      useLocalFilePath: !!attachmentsDir,
-    });
-    const stringifiedRecords = stringifier(records);
-    process.stdout.write(iconv.encode(stringifiedRecords, encoding));
   } catch (e) {
     logger.error(e);
     // eslint-disable-next-line no-process-exit

src/record/export/repositories/error.ts

@@ -0,0 +1,14 @@
+import { CliKintoneError } from "../../../utils/error";
+
+export class RepositoryError extends CliKintoneError {
+  constructor(cause: unknown) {
+    const message =
+      "An error occurred while exporting records to the data destination";
+    super(message, cause);
+
+    this.name = "RepositoryError";
+    // https://github.com/Microsoft/TypeScript/wiki/Breaking-Changes#extending-built-ins-like-error-array-and-map-may-no-longer-work
+    // Set the prototype explicitly.
+    Object.setPrototypeOf(this, RepositoryError.prototype);
+  }
+}

src/record/export/repositories/localRecordRepositoryFromStream.ts

@@ -0,0 +1,35 @@
+import type { LocalRecordRepository } from "../usecases/interface";
+import type { LocalRecord } from "../types/record";
+import type { RecordSchema } from "../types/schema";
+import { stringifierFactory } from "./stringifiers";
+
+export class LocalRecordRepositoryFromStream implements LocalRecordRepository {
+  private readonly destination: () => NodeJS.WritableStream;
+  readonly format = "csv";
+  readonly schema: RecordSchema;
+  readonly useLocalFilePath: boolean;
+
+  constructor(
+    destination: () => NodeJS.WritableStream,
+    schema: RecordSchema,
+    useLocalFilePath: boolean
+  ) {
+    this.destination = destination;
+    this.schema = schema;
+    this.useLocalFilePath = useLocalFilePath;
+  }
+  writer() {
+    const dest = this.destination();
+    const stringifier = stringifierFactory({
+      format: this.format,
+      schema: this.schema,
+      useLocalFilePath: this.useLocalFilePath,
+    });
+    return {
+      write: async (input: LocalRecord[]) => {
+        const recordsString = await stringifier.stringify(input);
+        dest.write(recordsString);
+      },
+    };
+  }
+}

src/record/export/repositories/localRecordRepositoryMock.ts

@@ -0,0 +1,32 @@
+import type { LocalRecordRepository } from "../usecases/interface";
+import type { LocalRecord } from "../types/record";
+import type { RecordSchema } from "../types/schema";
+import { stringifierFactory } from "./stringifiers";
+
+export class LocalRecordRepositoryMock implements LocalRecordRepository {
+  destination: string = "";
+  receivedRecords: LocalRecord[] = [];
+
+  readonly format = "csv";
+  readonly schema: RecordSchema;
+  readonly useLocalFilePath: boolean;
+
+  constructor(schema: RecordSchema, useLocalFilePath: boolean) {
+    this.schema = schema;
+    this.useLocalFilePath = useLocalFilePath;
+  }
+  writer() {
+    const stringifier = stringifierFactory({
+      format: this.format,
+      schema: this.schema,
+      useLocalFilePath: this.useLocalFilePath,
+    });
+    return {
+      write: async (input: LocalRecord[]) => {
+        this.receivedRecords.push(...input);
+        const recordsString = await stringifier.stringify(input);
+        this.destination += recordsString;
+      },
+    };
+  }
+}

src/record/export/stringifiers/__tests__/fixtures/input.ts → src/record/export/repositories/stringifiers/__tests__/fixtures/input.ts

@@ -1,6 +1,6 @@
-import type { KintoneRecord } from "../../../types/record";
+import type { LocalRecord } from "../../../../types/record";
 
-export const input: KintoneRecord[] = [
+export const input: LocalRecord[] = [
   {
     レコード番号: {
       type: "RECORD_NUMBER",

src/record/export/stringifiers/__tests__/fixtures/schema.ts → src/record/export/repositories/stringifiers/__tests__/fixtures/schema.ts

@@ -1,4 +1,4 @@
-import type { RecordSchema } from "../../../types/schema";
+import type { RecordSchema } from "../../../../types/schema";
 
 export const schema: RecordSchema = {
   hasSubtable: false,

src/record/export/stringifiers/__tests__/index.test.ts → src/record/export/repositories/stringifiers/__tests__/index.test.ts

@@ -2,22 +2,15 @@ import { input } from "./fixtures/input";
 import { schema } from "./fixtures/schema";
 import { stringifierFactory } from "../index";
 
-describe("printRecords", () => {
+describe("stringifier", () => {
   it("should print records as csv correctly", async () => {
     const { expected } = await import("./fixtures/expected_csv");
     const stringifier = stringifierFactory({
       format: "csv",
       schema,
       useLocalFilePath: false,
     });
-    const records = stringifier(input);
-    expect(records).toBe(expected);
-  });
-
-  it("should print records as json correctly", async () => {
-    const { expected } = await import("./fixtures/expected_json");
-    const stringifier = stringifierFactory({ format: "json" });
-    const records = stringifier(input);
+    const records = await stringifier.stringify(input);
     expect(records).toBe(expected);
   });
 });

src/record/export/stringifiers/stringifyAsCsv/__tests__/fieldValue.test.ts → src/record/export/repositories/stringifiers/csvStringifier/__tests__/fieldValue.test.ts

@@ -1,4 +1,4 @@
-import type * as Fields from "../../../types/field";
+import type * as Fields from "../../../../types/field";
 
 import { convertFieldValue } from "../fieldValue";
 

src/record/export/stringifiers/stringifyAsCsv/__tests__/fixtures/index/common/input.ts → src/record/export/repositories/stringifiers/csvStringifier/__tests__/fixtures/index/common/input.ts

@@ -1,6 +1,6 @@
-import type { KintoneRecord } from "../../../../../../types/record";
+import type { LocalRecord } from "../../../../../../../types/record";
 
-export const input: KintoneRecord[] = [
+export const input: LocalRecord[] = [
   {
     recordNumber: {
       type: "RECORD_NUMBER",

src/record/export/stringifiers/stringifyAsCsv/__tests__/fixtures/header/schema.ts → src/record/export/repositories/stringifiers/csvStringifier/__tests__/fixtures/index/common/schema.ts

@@ -1,4 +1,4 @@
-import type { RecordSchema } from "../../../../../types/schema";
+import type { RecordSchema } from "../../../../../../../types/schema";
 
 export const schema: RecordSchema = {
   hasSubtable: false,

src/record/export/stringifiers/stringifyAsCsv/__tests__/fixtures/index/withEmptySubtable/input.ts → src/record/export/repositories/stringifiers/csvStringifier/__tests__/fixtures/index/withEmptySubtable/input.ts

@@ -1,6 +1,6 @@
-import type { KintoneRecord } from "../../../../../../types/record";
+import type { LocalRecord } from "../../../../../../../types/record";
 
-export const input: KintoneRecord[] = [
+export const input: LocalRecord[] = [
   {
     recordNumber: {
       type: "RECORD_NUMBER",

src/record/export/stringifiers/stringifyAsCsv/__tests__/fixtures/index/withSubtableAndFileAndAttachmentsDir/schema.ts → src/record/export/repositories/stringifiers/csvStringifier/__tests__/fixtures/index/withEmptySubtable/schema.ts

@@ -1,4 +1,4 @@
-import type { RecordSchema } from "../../../../../../types/schema";
+import type { RecordSchema } from "../../../../../../../types/schema";
 
 export const schema: RecordSchema = {
   hasSubtable: true,

src/record/export/stringifiers/stringifyAsCsv/__tests__/fixtures/index/withFileAndAttachmentsDir/input.ts → src/record/export/repositories/stringifiers/csvStringifier/__tests__/fixtures/index/withFileAndAttachmentsDir/input.ts

@@ -1,6 +1,6 @@
-import type { KintoneRecord } from "../../../../../../types/record";
+import type { LocalRecord } from "../../../../../../../types/record";
 
-export const input: KintoneRecord[] = [
+export const input: LocalRecord[] = [
   {
     recordNumber: {
       type: "RECORD_NUMBER",

src/record/export/stringifiers/stringifyAsCsv/__tests__/fixtures/index/withFileAndNoAttachmentsDir/schema.ts → src/record/export/repositories/stringifiers/csvStringifier/__tests__/fixtures/index/withFileAndAttachmentsDir/schema.ts

@@ -1,4 +1,4 @@
-import type { RecordSchema } from "../../../../../../types/schema";
+import type { RecordSchema } from "../../../../../../../types/schema";
 
 export const schema: RecordSchema = {
   hasSubtable: false,

src/record/export/stringifiers/stringifyAsCsv/__tests__/fixtures/index/withFileAndNoAttachmentsDir/input.ts → src/record/export/repositories/stringifiers/csvStringifier/__tests__/fixtures/index/withFileAndNoAttachmentsDir/input.ts

@@ -1,6 +1,6 @@
-import type { KintoneRecord } from "../../../../../../types/record";
+import type { LocalRecord } from "../../../../../../../types/record";
 
-export const input: KintoneRecord[] = [
+export const input: LocalRecord[] = [
   {
     recordNumber: {
       type: "RECORD_NUMBER",

src/record/export/stringifiers/stringifyAsCsv/__tests__/fixtures/header/file_schema.ts → src/record/export/repositories/stringifiers/csvStringifier/__tests__/fixtures/index/withFileAndNoAttachmentsDir/schema.ts

@@ -1,4 +1,4 @@
-import type { RecordSchema } from "../../../../../types/schema";
+import type { RecordSchema } from "../../../../../../../types/schema";
 
 export const schema: RecordSchema = {
   hasSubtable: false,

src/record/export/stringifiers/stringifyAsCsv/__tests__/fixtures/index/withSubtableAndFileAndAttachmentsDir/input.ts → src/record/export/repositories/stringifiers/csvStringifier/__tests__/fixtures/index/withSubtableAndFileAndAttachmentsDir/input.ts

@@ -1,6 +1,6 @@
-import type { KintoneRecord } from "../../../../../../types/record";
+import type { LocalRecord } from "../../../../../../../types/record";
 
-export const input: KintoneRecord[] = [
+export const input: LocalRecord[] = [
   {
     recordNumber: {
       type: "RECORD_NUMBER",

src/record/export/stringifiers/stringifyAsCsv/__tests__/fixtures/index/withSubtableAndFileAndNoAttachmentsDir/schema.ts → src/record/export/repositories/stringifiers/csvStringifier/__tests__/fixtures/index/withSubtableAndFileAndAttachmentsDir/schema.ts

@@ -1,4 +1,4 @@
-import type { RecordSchema } from "../../../../../../types/schema";
+import type { RecordSchema } from "../../../../../../../types/schema";
 
 export const schema: RecordSchema = {
   hasSubtable: true,

src/record/export/stringifiers/stringifyAsCsv/__tests__/fixtures/index/withSubtableAndFileAndNoAttachmentsDir/input.ts → src/record/export/repositories/stringifiers/csvStringifier/__tests__/fixtures/index/withSubtableAndFileAndNoAttachmentsDir/input.ts

@@ -1,6 +1,6 @@
-import type { KintoneRecord } from "../../../../../../types/record";
+import type { LocalRecord } from "../../../../../../../types/record";
 
-export const input: KintoneRecord[] = [
+export const input: LocalRecord[] = [
   {
     recordNumber: {
       type: "RECORD_NUMBER",

src/record/export/stringifiers/stringifyAsCsv/__tests__/fixtures/header/subtable_schema.ts → src/record/export/repositories/stringifiers/csvStringifier/__tests__/fixtures/index/withSubtableAndFileAndNoAttachmentsDir/schema.ts

@@ -1,4 +1,4 @@
-import type { RecordSchema } from "../../../../../types/schema";
+import type { RecordSchema } from "../../../../../../../types/schema";
 
 export const schema: RecordSchema = {
   hasSubtable: true,

src/record/export/stringifiers/stringifyAsCsv/__tests__/index.test.ts → src/record/export/repositories/stringifiers/csvStringifier/__tests__/index.test.ts

@@ -1,7 +1,7 @@
-import type { KintoneRecord } from "../../../types/record";
-import type { RecordSchema } from "../../../types/schema";
+import type { LocalRecord } from "../../../../types/record";
+import type { RecordSchema } from "../../../../types/schema";
 
-import { stringifyAsCsv } from "../index";
+import { CsvStringifier } from "../index";
 
 import { pattern as common } from "./fixtures/index/common";
 import { pattern as withFileAndAttachmentsDir } from "./fixtures/index/withFileAndAttachmentsDir";
@@ -12,13 +12,13 @@ import { pattern as withEmptySubtable } from "./fixtures/index/withEmptySubtable
 
 export type TestPattern = {
   description: string;
-  input: KintoneRecord[];
+  input: LocalRecord[];
   schema: RecordSchema;
   useLocalFilePath: boolean;
   expected: string;
 };
 
-describe("stringifyAsCsv", () => {
+describe("csvStringifier", () => {
   const patterns: TestPattern[] = [
     common,
     withFileAndAttachmentsDir,
@@ -27,9 +27,12 @@ describe("stringifyAsCsv", () => {
     withSubtableAndFileAndNoAttachmentsDir,
     withEmptySubtable,
   ];
-  it.each(patterns)("$description", (pattern) => {
-    expect(
-      stringifyAsCsv(pattern.input, pattern.schema, pattern.useLocalFilePath)
-    ).toEqual(pattern.expected);
+  it.each(patterns)("$description", async (pattern) => {
+    const csvStringifier = new CsvStringifier(
+      pattern.schema,
+      pattern.useLocalFilePath
+    );
+    const actual = await csvStringifier.stringify(pattern.input);
+    expect(actual).toEqual(pattern.expected);
   });
 });