diff --git a/build/conf/ts/ts.conf b/build/conf/ts/ts.conf index 52d47a84..7c156848 100644 --- a/build/conf/ts/ts.conf +++ b/build/conf/ts/ts.conf @@ -11,6 +11,9 @@ TS_CONFIG_USE_OUTDIR= TS_USE_PREBUILT_NOTS_TOOL=yes NOTS_TOOL=${tool:"devtools/frontend_build_platform/nots/builder"} +# Additional commands that module can add (with `&&` as delimiter), if set those will be executed +# in build phase just before building the module. +_TS_PROJECT_SETUP_CMD= TS_CONFIG_PATH=tsconfig.json @@ -182,12 +185,14 @@ macro _AS_HIDDEN_INPUTS(IN{input}[]) { .CMD=${hide;context=TEXT;input:IN} } - -_TS_FILES_COPY_CMD= +# Input/output directives for files set by use with +# SRCS / TS_LARGE_FILES / TS_FILES / TS_FILES_GLOB +_TS_FILES_INOUTS= ### @usage: TS_FILES(Files...) ### -### Adds files to output as is. Similar to FILES but works for TS build modules +### Adds files to output as is. Does not add a command to copy the file to builddir. +### Similar to FILES but works for TS build modules ### Documentation: https://docs.yandex-team.ru/frontend-in-arcadia/references/TS_PACKAGE#ts-files macro TS_FILES(Files...) { # TODO: FBP-1795 @@ -207,7 +212,7 @@ macro TS_FILES_GLOB(Glob...) { ### ### Use large file ether from working copy or from remote storage via placeholder .external ### If is present locally (and not a symlink!) it will be copied to build directory. -### Otherwise macro will try to locate .external, parse it retrieve ot during build phase. +### Otherwise macro will try to locate .external, parse it and fetch the file during build phase. ### ### Then file will be copied to DESTINATION folder preserving file structure. ### Copied file becomes output of TS_PACKAGE diff --git a/build/conf/ts/ts_next.conf b/build/conf/ts/ts_next.conf index 5ee34d3e..cb2d8a2a 100644 --- a/build/conf/ts/ts_next.conf +++ b/build/conf/ts/ts_next.conf @@ -30,13 +30,13 @@ macro TS_NEXT_EXPERIMENTAL_COMPILE() { SET(TS_NEXT_COMMAND experimental-compile) } -TS_NEXT_CMD=$_TS_FILES_COPY_CMD \ +TS_NEXT_CMD=$_TS_PROJECT_SETUP_CMD \ && $ADD_VCS_INFO_FILE_CMD \ && $NOTS_TOOL $NOTS_TOOL_BASE_ARGS build-next $NOTS_TOOL_COMMON_BUILDER_ARGS \ --ts-next-command ${TS_NEXT_COMMAND} \ --bundler-config-path ${input:TS_NEXT_CONFIG_PATH} \ --output-dirs ${TS_NEXT_OUTPUT_DIR} \ - $_NODE_MODULES_INOUTS ${hide:PEERS} \ + $_NODE_MODULES_INOUTS $_TS_FILES_INOUTS ${hide:PEERS} \ ${hide;input:"package.json"} ${TS_CONFIG_FILES} $_AS_HIDDEN_INPUTS(IN $TS_INPUT_FILES) \ ${hide;output:"package.json"} \ ${hide;kv:"pc magenta"} ${hide;kv:"p TS_NXT"} diff --git a/build/conf/ts/ts_package.conf b/build/conf/ts/ts_package.conf index ca3d09a2..12dd298b 100644 --- a/build/conf/ts/ts_package.conf +++ b/build/conf/ts/ts_package.conf @@ -1,13 +1,12 @@ - TS_PACK=$TOUCH_UNIT \ + && $_TS_PROJECT_SETUP_CMD \ && $NOTS_TOOL $NOTS_TOOL_BASE_ARGS build-package $_NODE_MODULES_INOUTS \ - && $COPY_CMD ${input:"package.json"} ${output:"package.json"} \ - && $_TS_FILES_COPY_CMD \ - ${hide;kv:"p TS_PKG"} ${hide;kv:"pc magenta"} + $_TS_FILES_INOUTS ${hide;kv:"p TS_PKG"} ${hide;kv:"pc magenta"} ### # internal macro _TS_PACKAGE_EPILOGUE() { - _TS_PACKAGE_CHECK_FILES() + _TS_PACKAGE_CHECK_FILES() + TS_FILES(package.json) } ### @usage: TS_PACKAGE() diff --git a/build/conf/ts/ts_proto.conf b/build/conf/ts/ts_proto.conf index 78cf3285..7f127b10 100644 --- a/build/conf/ts/ts_proto.conf +++ b/build/conf/ts/ts_proto.conf @@ -1,12 +1,11 @@ _TS_PROTO_OPT= -_TS_PROTO_IMPL_CMD=$TOUCH_UNIT \ - && $_TS_FILES_COPY_CMD \ +_TS_PROTO_IMPL_CMD=$_TS_PROJECT_SETUP_CMD \ && $NOTS_TOOL $NOTS_TOOL_BASE_ARGS build-ts-proto $NOTS_TOOL_COMMON_BUILDER_ARGS \ ${_TS_PROTO_OPT} \ --protoc-bin $PROTOC \ --proto-srcs $_TS_PROTO_SRCS_FILES \ --proto-paths ./$PROTO_NAMESPACE $ARCADIA_ROOT/$PROTO_NAMESPACE $_PROTO__INCLUDE $ARCADIA_BUILD_ROOT $PROTOBUF_INCLUDE_PATH \ - $_NODE_MODULES_INOUTS \ + $_NODE_MODULES_INOUTS $_TS_FILES_INOUTS \ ${hide;input:"package.json"} ${TS_CONFIG_FILES} $_AS_HIDDEN_INPUTS(IN $TS_INPUT_FILES) \ ${hide;output:"package.json"} \ ${hide:PROTO_FAKEID} \ diff --git a/build/conf/ts/ts_tsc.conf b/build/conf/ts/ts_tsc.conf index 26a6e354..a49f6508 100644 --- a/build/conf/ts/ts_tsc.conf +++ b/build/conf/ts/ts_tsc.conf @@ -1,6 +1,6 @@ -TS_TSC_CMD=$_TS_FILES_COPY_CMD \ +TS_TSC_CMD=$_TS_PROJECT_SETUP_CMD \ && $NOTS_TOOL $NOTS_TOOL_BASE_ARGS build-tsc $NOTS_TOOL_COMMON_BUILDER_ARGS \ - $_NODE_MODULES_INOUTS ${hide:PEERS} \ + $_NODE_MODULES_INOUTS $_TS_FILES_INOUTS ${hide:PEERS} \ ${hide;input:"package.json"} ${TS_CONFIG_FILES} $_AS_HIDDEN_INPUTS(IN $TS_INPUT_FILES) \ ${hide;output:"package.json"} \ ${hide;kv:"pc magenta"} ${hide;kv:"p TS_TSC"} diff --git a/build/conf/ts/ts_vite.conf b/build/conf/ts/ts_vite.conf index e03739fb..3c80276e 100644 --- a/build/conf/ts/ts_vite.conf +++ b/build/conf/ts/ts_vite.conf @@ -32,13 +32,12 @@ macro VITE_OUTPUT(DirName) { SET(VITE_OUTPUT_DIR $DirName) } - -TS_VITE_CMD=$_TS_FILES_COPY_CMD \ +TS_VITE_CMD=$_TS_PROJECT_SETUP_CMD \ && $ADD_VCS_INFO_FILE_CMD \ && $NOTS_TOOL $NOTS_TOOL_BASE_ARGS build-vite $NOTS_TOOL_COMMON_BUILDER_ARGS \ --bundler-config-path ${input:VITE_CONFIG_PATH} \ --output-dirs ${VITE_OUTPUT_DIR} \ - $_NODE_MODULES_INOUTS ${hide:PEERS} \ + $_NODE_MODULES_INOUTS $_TS_FILES_INOUTS ${hide:PEERS} \ ${hide;input:"package.json"} ${TS_CONFIG_FILES} $_AS_HIDDEN_INPUTS(IN $TS_INPUT_FILES) \ ${hide;output:"package.json"} \ ${hide;kv:"pc magenta"} ${hide;kv:"p TS_VIT"} diff --git a/build/conf/ts/ts_webpack.conf b/build/conf/ts/ts_webpack.conf index 51e937fc..690adcb0 100644 --- a/build/conf/ts/ts_webpack.conf +++ b/build/conf/ts/ts_webpack.conf @@ -30,12 +30,12 @@ macro WEBPACK_OUTPUT(FirstDirName, DirNames...) { SET(WEBPACK_OUTPUT_DIR $FirstDirName $DirNames) } -TS_WEBPACK_CMD=$_TS_FILES_COPY_CMD \ +TS_WEBPACK_CMD=$_TS_PROJECT_SETUP_CMD \ && $ADD_VCS_INFO_FILE_CMD \ && $NOTS_TOOL $NOTS_TOOL_BASE_ARGS build-webpack $NOTS_TOOL_COMMON_BUILDER_ARGS \ --bundler-config-path ${input:WEBPACK_CONFIG_PATH} \ --output-dirs ${WEBPACK_OUTPUT_DIR} \ - $_NODE_MODULES_INOUTS ${hide:PEERS} \ + $_NODE_MODULES_INOUTS $_TS_FILES_INOUTS ${hide:PEERS} \ ${hide;input:"package.json"} ${TS_CONFIG_FILES} $_AS_HIDDEN_INPUTS(IN $TS_INPUT_FILES) \ ${hide;output:"package.json"} \ ${hide;kv:"pc magenta"} ${hide;kv:"p TS_WPK"} diff --git a/build/plugins/nots.py b/build/plugins/nots.py index 7dba60f6..f79707f2 100644 --- a/build/plugins/nots.py +++ b/build/plugins/nots.py @@ -310,6 +310,12 @@ def _build_cmd_input_paths(paths: list[str] | tuple[str], hide=False, disable_in return _build_directives([hide_part, disable_ip_part, "input"], paths) +def _build_cmd_output_paths(paths: list[str] | tuple[str], hide=False): + hide_part = "hide" if hide else "" + + return _build_directives([hide_part, "output"], paths) + + def _create_erm_json(unit: NotsUnitType): from lib.nots.erm_json_lite import ErmJsonLite @@ -517,15 +523,14 @@ def on_setup_build_env(unit: NotsUnitType) -> None: unit.set(["NOTS_TOOL_BUILD_ENV", " ".join(options)]) -def __set_append(unit: NotsUnitType, var_name: str, value: UnitType.PluginArgs) -> None: +def __set_append(unit: NotsUnitType, var_name: str, value: UnitType.PluginArgs, delimiter: str = " ") -> None: """ SET_APPEND() python naive implementation - append value/values to the list of values """ - previous_value = unit.get(var_name) or "" - value_in_str = " ".join(value) if isinstance(value, list) or isinstance(value, tuple) else value - new_value = previous_value + " " + value_in_str - - unit.set([var_name, new_value]) + old_str_value = unit.get(var_name) + old_values = [old_str_value] if old_str_value else [] + new_values = list(value) if isinstance(value, list) or isinstance(value, tuple) else [value] + unit.set([var_name, delimiter.join(old_values + new_values)]) def __strip_prefix(prefix: str, line: str) -> str: @@ -920,21 +925,24 @@ def on_set_ts_test_for_vars(unit: NotsUnitType, for_mod: str) -> None: unit.set(["TS_TEST_FOR_PATH", rootrel_arc_src(for_mod, unit)]) -@_with_report_configure_error -def on_ts_files(unit: NotsUnitType, *files: str) -> None: - for f in files: +def __on_ts_files(unit: NotsUnitType, files_in: list[str], files_out: list[str]) -> None: + for f in files_in: if f.startswith(".."): ymake.report_configure_error( - "Macro TS_FILES() does not allow to get files from parent directory.\n" + "TS_FILES* macroses are only allowed to use files inside the project directory.\n" f"Got path '{f}'.\n" "Docs: https://docs.yandex-team.ru/frontend-in-arcadia/references/TS_PACKAGE#ts-files." ) - new_cmds = ['$COPY_CMD ${{context=TEXT;input:"{0}"}} ${{noauto;output:"{0}"}}'.format(f) for f in files] - all_cmds = unit.get("_TS_FILES_COPY_CMD") - if all_cmds: - new_cmds.insert(0, all_cmds) - unit.set(["_TS_FILES_COPY_CMD", " && ".join(new_cmds)]) + new_items = _build_cmd_input_paths(paths=files_in, hide=True, disable_include_processor=True) + new_items += _build_cmd_output_paths(paths=files_out, hide=True) + __set_append(unit, "_TS_FILES_INOUTS", new_items) + logger.print_vars("_TS_FILES_INOUTS") + + +@_with_report_configure_error +def on_ts_files(unit: NotsUnitType, *files: str) -> None: + __on_ts_files(unit, files, files) @_with_report_configure_error @@ -951,21 +959,21 @@ def on_ts_large_files(unit: NotsUnitType, destination: str, *files: list[str]) - ) return + in_files = [os.path.join('${BINDIR}', f) for f in files] + out_files = [os.path.join('${BINDIR}', destination, f) for f in files] + # TODO: FBP-1795 # ${BINDIR} prefix for input is important to resolve to result of LARGE_FILES and not to SOURCEDIR - new_cmds = [ - '$COPY_CMD ${{context=TEXT;input:"${{BINDIR}}/{0}"}} ${{noauto;output:"{1}/{0}"}}'.format(f, destination) - for f in files - ] - all_cmds = unit.get("_TS_FILES_COPY_CMD") - if all_cmds: - new_cmds.insert(0, all_cmds) - unit.set(["_TS_FILES_COPY_CMD", " && ".join(new_cmds)]) + new_items = [f'$COPY_CMD {i} {o}' for (i, o) in zip(in_files, out_files)] + __set_append(unit, "_TS_PROJECT_SETUP_CMD", new_items, " && ") + logger.print_vars("_TS_PROJECT_SETUP_CMD") + + __on_ts_files(unit, in_files, out_files) @_with_report_configure_error def on_ts_package_check_files(unit: NotsUnitType) -> None: - ts_files = unit.get("_TS_FILES_COPY_CMD") + ts_files = unit.get("_TS_FILES_INOUTS") if ts_files == "": ymake.report_configure_error( "\n" diff --git a/devtools/frontend_build_platform/nots/builder/api/builders/base_builder.py b/devtools/frontend_build_platform/nots/builder/api/builders/base_builder.py index 241577f5..240323ac 100644 --- a/devtools/frontend_build_platform/nots/builder/api/builders/base_builder.py +++ b/devtools/frontend_build_platform/nots/builder/api/builders/base_builder.py @@ -16,12 +16,64 @@ ) from build.plugins.lib.nots.typescript import TsConfig from devtools.frontend_build_platform.libraries.logging import timeit -from ..models import BuildError, CommonBuildersOptions -from ..utils import copy_if_not_exists, extract_peer_tars, popen, resolve_bin +from ..models import BaseOptions, BuildError, CommonBuildersOptions +from ..utils import recursive_copy, extract_peer_tars, popen, resolve_bin + + +class BaseBuilderFileManager(object): + """ + Re-usaable basic functionality for managing src files. + One do not need to inherit it in builders - inherit BaseBuilder instead and customize the functions. + """ + + def __init__(self, options: BaseOptions, ts_config_path: str = "", output_dirs: list[str] = []): + self.options = options + self.ts_config_path = ts_config_path + self.output_dirs = output_dirs + + @timeit + def _get_copy_ignore_list(self) -> set[str]: + return { + # IDE's + ".idea", + ".vscode", + # Output dirs + "dist", + pm_constants.BUILD_DIRNAME, + pm_constants.BUNDLE_DIRNAME, + # Dependencies + pm_constants.NODE_MODULES_DIRNAME, + pm_constants.PNPM_LOCKFILE_FILENAME, + # ya-make artefacts + pm_constants.NODE_MODULES_WORKSPACE_BUNDLE_FILENAME, + "output.tar", # TODO FBP-1978 + pm_constants.OUTPUT_TAR_UUID_FILENAME, + # Other + ".traces", + "a.yaml", + self.ts_config_path, # Will be generated inside the builder (merged all the `extends`) + }.union(self.output_dirs) + + @timeit + def _copy_src_files_to_bindir(self): + ignore_list = self._get_copy_ignore_list() + for entry in os.scandir(self.options.curdir): + if entry.name in ignore_list: + continue + + dst = os.path.normpath(os.path.join(self.options.bindir, entry.name)) + recursive_copy(entry.path, dst) + + @timeit + def _copy_package_json(self): + shutil.copyfile( + pm_utils.build_pj_path(self.options.curdir), + pm_utils.build_pj_path(self.options.bindir), + ) @add_metaclass(ABCMeta) -class BaseBuilder(object): +class BaseBuilder(BaseBuilderFileManager): @staticmethod @timeit def load_ts_config(ts_config_file: str, sources_path: str) -> TsConfig: @@ -70,9 +122,7 @@ def __init__( :param external_dependencies: external dependencies which will be linked to node_modules/ (mapping name -> path) :type external_dependencies: dict """ - self.options = options - self.output_dirs = output_dirs - self.ts_config_path = ts_config_path + super(BaseBuilder, self).__init__(options=options, ts_config_path=ts_config_path, output_dirs=output_dirs) self.copy_package_json = copy_package_json self.external_dependencies = external_dependencies @@ -93,57 +143,14 @@ def resolve_bin(self, package_name: str, bin_name: str = None) -> str: @timeit def build(self): - self._copy_package_json() + if self.copy_package_json: + self._copy_package_json() self._prepare_dependencies() self._copy_src_files_to_bindir() self._build() self._run_javascript_after_build() - @timeit - def _get_copy_ignore_list(self) -> set[str]: - return { - # IDE's - ".idea", - ".vscode", - # Output dirs - "dist", - pm_constants.BUILD_DIRNAME, - pm_constants.BUNDLE_DIRNAME, - # Dependencies - pm_constants.NODE_MODULES_DIRNAME, - pm_constants.PNPM_LOCKFILE_FILENAME, - # ya-make artefacts - pm_constants.NODE_MODULES_WORKSPACE_BUNDLE_FILENAME, - "output.tar", # TODO FBP-1978 - pm_constants.OUTPUT_TAR_UUID_FILENAME, - # Other - ".traces", - "a.yaml", - self.ts_config_path, # Will be generated inside the builder (merged all the `extends`) - }.union(self.output_dirs) - - @timeit - def _copy_src_files_to_bindir(self): - ignore_list = self._get_copy_ignore_list() - - for entry in os.scandir(self.options.curdir): - if entry.name in ignore_list: - continue - - dst = os.path.normpath(os.path.join(self.options.bindir, entry.name)) - copy_if_not_exists(entry.path, dst) - - @timeit - def _copy_package_json(self): - if not self.copy_package_json: - return - - shutil.copyfile( - pm_utils.build_pj_path(self.options.curdir), - pm_utils.build_pj_path(self.options.bindir), - ) - @timeit def __extract_peer_tars(self, *args, **kwargs): return extract_peer_tars(*args, **kwargs) diff --git a/devtools/frontend_build_platform/nots/builder/api/utils.py b/devtools/frontend_build_platform/nots/builder/api/utils.py index f9470c0c..4a2387bb 100644 --- a/devtools/frontend_build_platform/nots/builder/api/utils.py +++ b/devtools/frontend_build_platform/nots/builder/api/utils.py @@ -2,6 +2,7 @@ import shutil import stat import subprocess +import sys import library.python.archive as archive @@ -13,6 +14,10 @@ from devtools.frontend_build_platform.libraries.logging import timeit +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + + @timeit def extract_peer_tars(moddir_abs: str, visited: set[str] = set()): """Extracts all the output tars for the dependency tree, excluding root @@ -66,16 +71,26 @@ def _extract_output_tar(moddir_abs: str): @timeit -def __copy_file_with_write_permissions(src, dst): - os.makedirs(os.path.dirname(dst), exist_ok=True) - - shutil.copy(src, dst) +def __add_write_permissions(path): + if not os.path.exists(path): + eprint(f"Directory not exists: {path}") + return - dst_stat = os.stat(dst) + dst_stat = os.stat(path) dst_mode = stat.S_IMODE(dst_stat.st_mode) upd_mode = dst_mode | stat.S_IWUSR | stat.S_IWGRP if dst_mode != upd_mode: - os.chmod(dst, upd_mode) + try: + os.chmod(path, upd_mode) + except PermissionError: + eprint(f"Can't update permissions for {path}") + + +@timeit +def __copy_file_with_write_permissions(src, dst): + os.makedirs(os.path.dirname(dst), exist_ok=True) + shutil.copy(src, dst) + __add_write_permissions(dst) @timeit @@ -91,6 +106,21 @@ def copy_if_not_exists(src: str, dst: str): __copy_file_with_write_permissions(src, dst) +@timeit +def recursive_copy(src, dest, overwrite=False): + __add_write_permissions(os.path.dirname(dest)) + + if os.path.isdir(src): + os.makedirs(dest, exist_ok=True) + files = os.listdir(src) + for f in files: + recursive_copy(os.path.join(src, f), os.path.join(dest, f), overwrite) + + if os.path.isfile(src): + if not os.path.exists(dest) or overwrite: + __copy_file_with_write_permissions(src, dest) + + @timeit def simplify_colors(data): """ diff --git a/devtools/frontend_build_platform/nots/builder/cli/commands/build_package.py b/devtools/frontend_build_platform/nots/builder/cli/commands/build_package.py index 964e3089..e782bfa3 100644 --- a/devtools/frontend_build_platform/nots/builder/cli/commands/build_package.py +++ b/devtools/frontend_build_platform/nots/builder/cli/commands/build_package.py @@ -7,6 +7,7 @@ ) from devtools.frontend_build_platform.libraries.logging import timeit from devtools.frontend_build_platform.nots.builder.api import BaseOptions, create_node_modules +from devtools.frontend_build_platform.nots.builder.api.builders.base_builder import BaseBuilderFileManager @dataclass @@ -26,6 +27,10 @@ def build_package_parser(subparsers) -> ArgumentParser: @timeit def build_package_func(args: PackageBuilderOptions): + builder = BaseBuilderFileManager(args) + pj = PackageJson.load(pm_utils.build_pj_path(args.curdir)) if pj.has_dependencies(): create_node_modules(args) + + builder._copy_src_files_to_bindir() diff --git a/devtools/frontend_build_platform/nots/builder/resources.json b/devtools/frontend_build_platform/nots/builder/resources.json index af883432..e4f17a74 100644 --- a/devtools/frontend_build_platform/nots/builder/resources.json +++ b/devtools/frontend_build_platform/nots/builder/resources.json @@ -1,24 +1,24 @@ { "info": { - "created": "2025-02-03T06:37:39.820Z", + "created": "2025-02-17T19:39:00.113Z", "pr": { - "id": 7786036, - "author": "zaverden", - "summary": "feat(conf+builder): RUN_JAVASCRIPT_AFTER_BUILD" + "id": 7920090, + "author": "vturov", + "summary": "FBP-2079: Add COPY CMD in TS_FILES only for TS_PACKAGE" } }, "by_platform": { "darwin": { - "uri": "sbr:7957514369" + "uri": "sbr:8067392225" }, "darwin-arm64": { - "uri": "sbr:7957513860" + "uri": "sbr:8067390844" }, "linux": { - "uri": "sbr:7957515156" + "uri": "sbr:8067394227" }, "linux-aarch64": { - "uri": "sbr:7957514773" + "uri": "sbr:8067393420" } } }