diff --git a/docs/mkdocs.yaml b/docs/mkdocs.yaml index d02f5ffd9..0d2c89a71 100644 --- a/docs/mkdocs.yaml +++ b/docs/mkdocs.yaml @@ -11,22 +11,34 @@ nav: - configuring-ci-cd.md - versioning.md - API: - - makes.nix: - - api/makes.nix/index.md - - api/makes.nix/database.md - - api/makes.nix/deploy.md - - api/makes.nix/development.md - - api/makes.nix/environment.md - - api/makes.nix/examples.md - - api/makes.nix/format.md - - api/makes.nix/framework-configuration.md - - api/makes.nix/lint.md - - api/makes.nix/monitoring.md - - api/makes.nix/performance.md - - api/makes.nix/secrets.md - - api/makes.nix/security.md - - api/makes.nix/test.md - - api/makes.nix/utilities.md + - builtins: + - api/builtins/index.md + - api/builtins/database.md + - api/builtins/deploy.md + - api/builtins/development.md + - api/builtins/environment.md + - api/builtins/examples.md + - api/builtins/format.md + - api/builtins/framework-configuration.md + - api/builtins/lint.md + - api/builtins/monitoring.md + - api/builtins/performance.md + - api/builtins/secrets.md + - api/builtins/security.md + - api/builtins/test.md + - api/builtins/utilities.md + - extensions: + - api/extensions/index.md + - api/extensions/fundamentals.md + - api/extensions/containers.md + - api/extensions/fetchers.md + - api/extensions/format-conversion.md + - api/extensions/git.md + - api/extensions/node.js.md + - api/extensions/others.md + - api/extensions/patchers.md + - api/extensions/python.md + - api/extensions/ruby.md - architecture.md - contributing.md - governance.md diff --git a/docs/src/api/makes.nix/database.md b/docs/src/api/builtins/database.md similarity index 100% rename from docs/src/api/makes.nix/database.md rename to docs/src/api/builtins/database.md diff --git a/docs/src/api/makes.nix/deploy.md b/docs/src/api/builtins/deploy.md similarity index 99% rename from docs/src/api/makes.nix/deploy.md rename to docs/src/api/builtins/deploy.md index 99f51a394..6d681cce5 100644 --- a/docs/src/api/makes.nix/deploy.md +++ b/docs/src/api/builtins/deploy.md @@ -393,5 +393,5 @@ Example invocation: `$ m . /deployNomad/default/job1` Example invocation: `$ m . /deployNomad/staging/job2` -- [makes_environment]: ./environment.md -- [makes_secrets]: ./secrets.md +[makes_environment]: ./environment.md +[makes_secrets]: ./secrets.md diff --git a/docs/src/api/makes.nix/development.md b/docs/src/api/builtins/development.md similarity index 100% rename from docs/src/api/makes.nix/development.md rename to docs/src/api/builtins/development.md diff --git a/docs/src/api/makes.nix/environment.md b/docs/src/api/builtins/environment.md similarity index 98% rename from docs/src/api/makes.nix/environment.md rename to docs/src/api/builtins/environment.md index 9c45853fa..921455993 100644 --- a/docs/src/api/makes.nix/environment.md +++ b/docs/src/api/builtins/environment.md @@ -94,4 +94,4 @@ Example invocation: `$ m . /envVarsForTerraform/example` Example invocation: `$ m . /envVarsForTerraform/otherExample` -- [makes_secrets]: ./secrets.md +[makes_secrets]: ./secrets.md diff --git a/docs/src/api/makes.nix/examples.md b/docs/src/api/builtins/examples.md similarity index 100% rename from docs/src/api/makes.nix/examples.md rename to docs/src/api/builtins/examples.md diff --git a/docs/src/api/makes.nix/format.md b/docs/src/api/builtins/format.md similarity index 100% rename from docs/src/api/makes.nix/format.md rename to docs/src/api/builtins/format.md diff --git a/docs/src/api/makes.nix/framework-configuration.md b/docs/src/api/builtins/framework-configuration.md similarity index 100% rename from docs/src/api/makes.nix/framework-configuration.md rename to docs/src/api/builtins/framework-configuration.md diff --git a/docs/src/api/makes.nix/index.md b/docs/src/api/builtins/index.md similarity index 85% rename from docs/src/api/makes.nix/index.md rename to docs/src/api/builtins/index.md index 8091e8609..394376244 100644 --- a/docs/src/api/makes.nix/index.md +++ b/docs/src/api/builtins/index.md @@ -40,6 +40,6 @@ In the next sections we document all configuration options you can tweak in a `makes.nix`. -- [makes_environment]: #environment -- [makes_secrets]: #secrets -- [sops]: https://github.com/mozilla/sops +[makes_environment]: #environment +[makes_secrets]: #secrets +[sops]: https://github.com/mozilla/sops diff --git a/docs/src/api/makes.nix/lint.md b/docs/src/api/builtins/lint.md similarity index 100% rename from docs/src/api/makes.nix/lint.md rename to docs/src/api/builtins/lint.md diff --git a/docs/src/api/makes.nix/monitoring.md b/docs/src/api/builtins/monitoring.md similarity index 100% rename from docs/src/api/makes.nix/monitoring.md rename to docs/src/api/builtins/monitoring.md diff --git a/docs/src/api/makes.nix/performance.md b/docs/src/api/builtins/performance.md similarity index 100% rename from docs/src/api/makes.nix/performance.md rename to docs/src/api/builtins/performance.md diff --git a/docs/src/api/makes.nix/secrets.md b/docs/src/api/builtins/secrets.md similarity index 99% rename from docs/src/api/makes.nix/secrets.md rename to docs/src/api/builtins/secrets.md index be5dd3eb5..8d960a7c9 100644 --- a/docs/src/api/makes.nix/secrets.md +++ b/docs/src/api/builtins/secrets.md @@ -86,7 +86,7 @@ Example `makes.nix`: ## secretsForAwsFromGitlab Aquire an AWS session -using [Gitlab CI OIDC](https://docs.gitlab.com/ee/ci/cloud_services/aws/index.html). +using [GitLab CI OIDC](https://docs.gitlab.com/ee/ci/cloud_services/aws/index.html). Types: diff --git a/docs/src/api/makes.nix/security.md b/docs/src/api/builtins/security.md similarity index 100% rename from docs/src/api/makes.nix/security.md rename to docs/src/api/builtins/security.md diff --git a/docs/src/api/makes.nix/test.md b/docs/src/api/builtins/test.md similarity index 100% rename from docs/src/api/makes.nix/test.md rename to docs/src/api/builtins/test.md diff --git a/docs/src/api/makes.nix/utilities.md b/docs/src/api/builtins/utilities.md similarity index 97% rename from docs/src/api/makes.nix/utilities.md rename to docs/src/api/builtins/utilities.md index af6fc2ac4..e957f3c01 100644 --- a/docs/src/api/makes.nix/utilities.md +++ b/docs/src/api/builtins/utilities.md @@ -64,7 +64,7 @@ m github:fluidattacks/makes@23.04 /utils/makeRubyLock \ - Supported `ruby_version`s are: `2.7`, `3.0` and `3.1`. - `dependencies_yaml` is the **absolute path** to a YAML file - mapping [RubyGems](https://rubygems.org/gems/slim) gems to version constraints. + mapping [RubyGems](https://rubygems.org/) gems to version constraints. Example: diff --git a/docs/src/api/extensions/containers.md b/docs/src/api/extensions/containers.md new file mode 100644 index 000000000..be4197a5d --- /dev/null +++ b/docs/src/api/extensions/containers.md @@ -0,0 +1,119 @@ +## makeContainerImage + +Build a container image +in [OCI Format](https://github.com/opencontainers/image-spec). + +A container image is composed of: + +- 0 or more layers (binary blobs). + - Each layer contains a snapshot of the root file system (`/`), + they represent portions of it. + - When the container is executed + all layers are squashed together + to compose the root + of the file system (`/`). +- A JSON manifest (metadata) + that describes important aspects of the container, + for instance its layers, environment variables, entrypoint, etc. + +Resources: + +- https://grahamc.com/blog/nix-and-layered-docker-images + +Types: + +- makeContainerImage (`function { ... } -> package`): + - layers (`listOf package`): Optional. + Layers of the container. + Defaults to `[ ]`. + - maxLayers (`ints.positive`): Optional. + Maximum number of layers the container can have. + Defaults to `65`. + - config (`attrsOf anything`): Optional. + Configuration manifest as described in + [OCI Runtime Configuration Manifest](https://github.com/moby/moby/blob/master/image/spec/v1.2.) + Defaults to `{ }`. + +Example: + +```nix +# /path/to/my/project/makes/example/main.nix +{ inputs +, makeContainerImage +, makeDerivation +, ... +}: +makeContainerImage { + config = { + Env = [ + # Do not use this for sensitive values, it's not safe. + "EXAMPLE_ENV_VAR=example-value" + ]; + WorkingDir = "/working-dir"; + }; + layers = [ + inputs.nixpkgs.coreutils # ls, cat, etc + (makeDerivation { + name = "custom-layer"; + builder = '' + # $out represents the final container root file system: / + # + # The following commands are equivalent in Docker to: + # RUN mkdir /working-dir + # RUN echo my-file-contents > /working-dir/my-file + # + mkdir -p $out/working-dir + echo my-file-contents > $out/working-dir/my-file + ''; + }) + ]; +} +``` + +```bash +$ m . /example + + Creating layer 1 from paths: ['/nix/store/zqaqyidzsqc7z03g4ajgizy2lz1m19xz-libunistring-0.9.10'] + Creating layer 2 from paths: ['/nix/store/xjjdyb66g3cxd5880zspazsp5f16lbxz-libidn2-2.3.1'] + Creating layer 3 from paths: ['/nix/store/wvgyhnd3rn6dhxzbr5r71gx2q9mhgshj-glibc-2.32-48'] + Creating layer 4 from paths: ['/nix/store/ip0pxdd49l1v3cmxsvw8ziwmqhyzg5pf-attr-2.4.48'] + Creating layer 5 from paths: ['/nix/store/26vpasbj38nhj462kqclwp2i6s3hhdba-acl-2.3.1'] + Creating layer 6 from paths: ['/nix/store/937f5738d2frws07ixcpg5ip176pfss1-coreutils-8.32'] + Creating layer 7 from paths: ['/nix/store/fc24830z8lqa657grb3snvjjv9vxs7ql-custom-layer'] + Creating layer 8 with customisation... + Adding manifests... + Done. + + /nix/store/dvif4xy1l0qsjblxvzzcr6map1hg22w5-container-image.tar.gz + +$ docker load < /nix/store/dvif4xy1l0qsjblxvzzcr6map1hg22w5-container-image.tar.gz + + b5507f5bda26: Loading layer 133.1kB/133.1kB + da2b3a66ea19: Loading layer 1.894MB/1.894MB + eb4c566a2922: Loading layer 10.24kB/10.24kB + 19b7be559bbc: Loading layer 61.44kB/61.44kB + Loaded image: container-image:latest + +$ docker run container-image:latest pwd + + /working-dir + +$ docker run container-image:latest ls . + + my-file + +$ docker run container-image:latest cat my-file + + my-file-contents + +$ docker run container-image:latest ls / + + bin + dev + etc + libexec + nix + proc + sys + working-dir +``` diff --git a/docs/src/api/extensions/fetchers.md b/docs/src/api/extensions/fetchers.md new file mode 100644 index 000000000..0d5acc806 --- /dev/null +++ b/docs/src/api/extensions/fetchers.md @@ -0,0 +1,207 @@ +## fetchUrl + +Fetch a file from the specified URL. + +Types: + +- fetchUrl (`function { ... } -> package`): + + - url (`str`): + URL to download. + - sha256 (`str`): + SHA256 of the expected output, + In order to get the SHA256 + you can omit this parameter and execute Makes, + Makes will tell you the correct SHA256 on failure. + +Example: + +```nix +# /path/to/my/project/makes/example/main.nix +{ fetchUrl +, ... +}: +fetchUrl { + url = "https://github.com/fluidattacks/makes/blob/16aafa1e3ed4cc99eb354842341fbf6f478a211c/README.md"; + sha256 = "18scrymrar0bv7s92hfqfb01bv5pibyjw6dxp3i8nylmnh6gjv15"; +} + +``` + +## fetchArchive + +Fetch a Zip (.zip) or Tape Archive (.tar) from the specified URL +and unpack it. + +Types: + +- fetchArchive (`function { ... } -> package`): + + - url (`str`): + URL to download. + - sha256 (`str`): + SHA256 of the expected output, + In order to get the SHA256 + you can omit this parameter and execute Makes, + Makes will tell you the correct SHA256 on failure. + - stripRoot (`bool`): Optional. + Most archives have a symbolic top-level directory + that is discarded during unpack phase. + If this is not the case you can set this flag to `false`. + Defaults to `true`. + +Example: + +```nix +# /path/to/my/project/makes/example/main.nix +{ fetchArchive +, ... +}: +fetchArchive { + url = "https://github.com/fluidattacks/makes/archive/16aafa1e3ed4cc99eb354842341fbf6f478a211c.zip"; + sha256 = "16zx89lzv5n048h5l9f8dgpvdj0l38hx7aapc7h1d1mjc1ca2i6a"; +} + +``` + +## fetchGithub + +Fetch a commit from the specified Git repository at GitHub. + +Types: + +- fetchGithub (`function { ... } -> package`): + + - owner (`str`): + Owner of the repository. + - repo (`str`): + Name of the repository. + - rev (`str`): + Commit, branch or tag to fetch. + - sha256 (`str`): + SHA256 of the expected output, + In order to get the SHA256 + you can omit this parameter and execute Makes, + Makes will tell you the correct SHA256 on failure. + +Example: + +```nix +# /path/to/my/project/makes/example/main.nix +{ fetchGithub +, ... +}: +fetchGithub { + owner = "kamadorueda"; + repo = "mailmap-linter"; + rev = "e0799aa47ac5ce6776ca8581ba50ace362e5d0ce"; + sha256 = "02nr39rn4hicfam1rccbqhn6w6pl25xq7fl2kw0s0ahxzvfk24mh"; +} +``` + +## fetchGitlab + +Fetch a commit from the specified Git repository at GitLab. + +Types: + +- fetchGitlab (`function { ... } -> package`): + + - owner (`str`): + Owner of the repository. + - repo (`str`): + Name of the repository. + - rev (`str`): + Commit, branch or tag to fetch. + - sha256 (`str`): + SHA256 of the expected output, + In order to get the SHA256 + you can omit this parameter and execute Makes, + Makes will tell you the correct SHA256 on failure. + +Example: + +```nix +# /path/to/my/project/makes/example/main.nix +{ fetchGitlab +, ... +}: +fetchGitlab { + owner = "fluidattacks"; + repo = "product"; + rev = "ff231a9bf8aa3f0807f3431b402e7af08d136341"; + sha256 = "1sfbif0bchdpw4rlfpv9gs4l4bmg8l24fqh2hg6m39msrvh1w6h3"; +} +``` + +## fetchNixpkgs + +Fetch a commit from the +[Nixpkgs](https://github.com/NixOS/nixpkgs) repository. + +:warning: By default all licenses in the Nixpkgs repository are accepted. +Options to decline individual licenses are provided below. + +Types: + +- fetchNixpkgs (`function { ... } -> anything`): + - rev (`str`): + Commit, branch or tag to fetch. + - allowUnfree (`bool`): Optional. + Allow software that do not respect the freedom of its users. + Defaults to `true`. + - acceptAndroidSdkLicense (`bool`): Optional. + Accept the Android SDK license. + Defaults to `true`. + - overalys (`listOf overlayType`): Optional. + Overlays to apply to the Nixpkgs set. + Defaults to `[ ]`. + - sha256 (`str`): + SHA256 of the expected output, + In order to get the SHA256 + you can omit this parameter and execute Makes, + Makes will tell you the correct SHA256 on failure. + +Example: + +```nix +# /path/to/my/project/makes/example/main.nix +{ fetchNixpkgs +, ... +}: +let nixpkgs = fetchNixpkgs { + rev = "f88fc7a04249cf230377dd11e04bf125d45e9abe"; + sha256 = "1dkwcsgwyi76s1dqbrxll83a232h9ljwn4cps88w9fam68rf8qv3"; +}; +in +nixpkgs.awscli +``` + +## fetchRubyGem + +Fetch a Ruby gem +from [RubyGems](https://rubygems.org/). + +Types: + +- fetchRubyGem (`function { ... } -> package`): + - sha256 (`str`): + SHA256 of the expected output, + In order to get the SHA256 + you can omit this parameter and execute Makes, + Makes will tell you the correct SHA256 on failure. + - url (`str`): + url of the gem to download. + +Example: + +```nix +# /path/to/my/project/makes/example/main.nix +{ fetchRubyGem +, ... +}: +fetchRubyGem { + sha256 = "04nc8x27hlzlrr5c2gn7mar4vdr0apw5xg22wp6m8dx3wqr04a0y"; + url = "https://rubygems.org/downloads/ast-2.4.2.gem"; +} +``` diff --git a/docs/src/api/extensions/format-conversion.md b/docs/src/api/extensions/format-conversion.md new file mode 100644 index 000000000..04e457a99 --- /dev/null +++ b/docs/src/api/extensions/format-conversion.md @@ -0,0 +1,388 @@ +## fromJson + +Convert a JSON formatted string +to a Nix expression. + +Types: + +- fromJson (`function str -> anything`): + + - (`str`): + JSON formatted string to convert. + +Examples: + +```nix +# /path/to/my/project/makes/example/main.nix +{ fromJson +, makeDerivation +, ... +}: +let + data = fromJson '' + { + "name": "John", + "lastName": "Doe", + "tickets": 3 + } + ''; +in +makeDerivation { + env = { + envName = data.name; + envLastName = data.lastName; + envTickets = data.tickets; + }; + builder = '' + info "Name is: $envName" + info "Last name is: $envLastName" + info "Tickets is: $envTickets" + ''; + name = "example"; +} +``` + +```bash +$ m . /example + + [INFO] Name is: John + [INFO] Last name is: Doe + [INFO] Tickets is: 3 +``` + +## fromToml + +Convert a TOML formatted string +to a Nix expression. + +Types: + +- fromToml (`function str -> anything`): + + - (`str`): + TOML formatted string to convert. + +Examples: + +```nix +# /path/to/my/project/makes/example/main.nix +{ fromToml +, makeDerivation +, ... +}: +let + data = fromToml '' + [example] + name = "John" + lastName = "Doe" + tickets = 3 + ''; +in +makeDerivation { + env = { + envName = data.example.name; + envLastName = data.example.lastName; + envTickets = data.example.tickets; + }; + builder = '' + info "Name is: $envName" + info "Last name is: $envLastName" + info "Tickets is: $envTickets" + ''; + name = "example"; +} +``` + +```bash +$ m . /example + + [INFO] Name is: John + [INFO] Last name is: Doe + [INFO] Tickets is: 3 +``` + +## fromYaml + +Convert a YAML formatted string +to a Nix expression. + +Types: + +- fromYaml (`function str -> anything`): + + - (`str`): + YAML formatted string to convert. + +Examples: + +```nix +# /path/to/my/project/makes/example/main.nix +{ fromYaml +, makeDerivation +, ... +}: +let + data = fromYaml '' + name: "John" + lastName: "Doe" + tickets: 3 + ''; +in +makeDerivation { + env = { + envName = data.name; + envLastName = data.lastName; + envTickets = data.tickets; + }; + builder = '' + info "Name is: $envName" + info "Last name is: $envLastName" + info "Tickets is: $envTickets" + ''; + name = "example"; +} +``` + +```bash +$ m . /example + + [INFO] Name is: John + [INFO] Last name is: Doe + [INFO] Tickets is: 3 +``` + +## toBashArray + +Transform a list of arguments +into a Bash array. +It can be used for passing +several arguments from Nix +to Bash. + +Types: + +- toBashArray (`function (listOf strLike) -> package`): + + - (`listOf strLike`): + list of arguments + to transform. + +Examples: + +```nix +# /path/to/my/project/makes/example/main.nix +{ toBashArray +, makeDerivation +, ... +}: +makeDerivation { + env = { + envTargets = toBashArray [ "first" "second" "third" ]; + }; + builder = '' + source "$envTargets/template" export targets + for target in "''${targets[@]}"; do + info "$target" + info --- + done + ''; + name = "example"; +} +``` + +```bash +$ m . /example + + [INFO] first + [INFO] --- + [INFO] second + [INFO] --- + [INFO] third + [INFO] ---- +``` + +## toBashMap + +Transform a Nix `attrsOf strLike` expression +into a Bash associative array (map). +It can be used for passing +several arguments from Nix +to Bash. +You can combine with toBashArray for more complex structures. + +Types: + +- toBashMap (`function (attrsOf strLike) -> package`): + + - (`attrsOf strLike`): + expression to transform. + +Examples: + +```nix +# /path/to/my/project/makes/example/main.nix +{ toBashMap +, makeDerivation +, ... +}: +makeDerivation { + env = { + envData = toBashMap { + name = "Makes"; + tags = "ci/cd, framework, nix"; + }; + }; + builder = '' + source "$envData/template" data + + for target in "''${!targets[@]}"; do + info "$target" + info --- + done + ''; + name = "example"; +} +``` + +```bash +$ m . /example + + [INFO] key: tags + [INFO] value: ci/cd, framework, nix + [INFO] --- + [INFO] key: name + [INFO] value: Makes + [INFO] --- +``` + +## toFileJson + +Convert a Nix expression +into a JSON file. + +Types: + +- toFileJson (`function str anything -> package`): + + - (`str`): + Name of the created file. + - (`anything`): + Nix expression to convert. + +Examples: + +```nix +# /path/to/my/project/makes/example/main.nix +{ toFileJson +, makeDerivation +, ... +}: +makeDerivation { + env = { + envFile = toFileJson "example.json" { name = "value"; }; + }; + builder = '' + cat $envFile + ''; + name = "example"; +} +``` + +```bash +$ m . /example + + {"name": "value"} +``` + +## toFileJsonFromFileYaml + +Use [yq](https://github.com/mikefarah/yq) +to transform a YAML file +into its JSON +equivalent. + +Types: + +- toFileJsonFromFileYaml (`function package -> package`): + + - (`package`): + YAML file to transform. + +Examples: + +```yaml +# /path/to/my/project/makes/example/test.yaml + +name: "John" +lastName: "Doe" +tickets: 3 +``` + +```nix +# /path/to/my/project/makes/example/main.nix +{ makeDerivation +, projectPath +, toFileJsonFromFileYaml +, ... +}: +makeDerivation { + env = { + envJson = + toFileJsonFromFileYaml + (projectPath "/makes/example/test.yaml"); + }; + builder = '' + cat "$envJson" + ''; + name = "example"; +} +``` + +```bash +$ m . /example + +{ + "name": "John", + "lastName": "Doe", + "tickets": 3 +} +``` + +## toFileYaml + +Convert a Nix expression +into a YAML file. + +Types: + +- toFileYaml (`function str anything -> package`): + + - (`str`): + Name of the created file. + - (`anything`): + Nix expression to convert. + +Examples: + +```nix +# /path/to/my/project/makes/example/main.nix +{ toFileYaml +, makeDerivation +, ... +}: +makeDerivation { + env = { + envFile = toFileYaml "example.yaml" { name = "value"; }; + }; + builder = '' + cat $envFile + ''; + name = "example"; +} +``` + +```bash +$ m . /example + + name: value +``` diff --git a/docs/src/api/extensions/fundamentals.md b/docs/src/api/extensions/fundamentals.md new file mode 100644 index 000000000..17e6bba60 --- /dev/null +++ b/docs/src/api/extensions/fundamentals.md @@ -0,0 +1,663 @@ +## Main.nix format + +Each `main.nix` file under the `makes/` directory +should be a function that receives one or more arguments +and returns a derivation: + +```nix +{ argA +, argB +, ... +}: +doSomethingAndReturnADerivation +``` + +## Derivations + +On Nix +a [derivation](https://nixos.org/manual/nix/unstable/expressions/derivations.html) +is the process of: + +- taking zero or more inputs + +- transforming them as we see fit + +- placing the results in the output path + +Derivation outputs live in the `/nix/store`. +Their locations in the filesystem are always in the form: +`/nix/store/hash123-name` where +`hash123` is computed by hashing the derivation's inputs. + +Derivation outputs are: + +- A regular file +- A regular directory that contains arbitrary contents + +For instance the derivation output for Bash is: +`/nix/store/kxj6cblcsd1qcbbxlmbswwrn89zcmgd6-bash-4.4-p23` +which contains, among other files: + +```tree +/nix/store/kxj6cblcsd1qcbbxlmbswwrn89zcmgd6-bash-4.4-p23 +├── bin +│   ├── bash +│   └── sh +``` + +## makeSearchPaths + +On Linux +software dependencies +can be located anywhere in the file system. + +We can control where +programs find other programs, +dependencies, libraries, etc, +through special environment variables. + +Below we describe shortly the purpose +of the environment variables we currently support. + +- [CLASSPATH](https://www.javatpoint.com/how-to-set-classpath-in-java): + Location of user-defined classes and packages. + +- [CRYSTAL_LIBRARY_PATH](https://crystal-lang.org/reference/guides/static_linking.html): + Location of Crystal libraries. + +- [GEM_PATH](https://guides.rubygems.org/command-reference): + Location of libraries for Ruby. + +- [LD_LIBRARY_PATH](https://linuxhint.com/what-is-ld-library-path/): + Location of libraries for Dynamic Linking Loaders. + +- [MYPYPATH](https://mypy.readthedocs.io/en/stable/running_mypy.html): + Location of library stubs and static types + for [MyPy](https://mypy.readthedocs.io/en/stable/). + +- [NODE_PATH](https://nodejs.org/api/modules.html): + Location of Node.js modules. + +- [OCAMLPATH](https://github.com/ocaml/ocaml/issues/8898): + Location of OCaml libraries. + +- [CAML_LD_LIBRARY_PATH](https://ocaml.org/manual/runtime.html): + Location of OCaml stublibs. + +- [PATH](https://opensource.com/article/17/6/set-path-linux): + Location of directories where executable programs are located. + +- [PKG_CONFIG_PATH](https://linux.die.net/man/1/pkg-config): + Location of pkg-config packages. + +- [PYTHONPATH](https://docs.python.org/3/using/cmdline.html#envvar-PYTHONPATH): + Location of Python modules and site-packages. + +`makeSearchPaths` helps you write code like this: + +```nix +makeSearchPaths { + bin = [ inputs.nixpkgs.git ]; +} +``` + +Instead of this: + +```bash +export PATH="/nix/store/m5kp2jhiga25ynk3iq61f4psaqixg7ib-git-2.32.0/bin${PATH:+:}${PATH:-}" +``` + +Types: + +- makeSearchPaths (`function { ... } -> package`): + + - `bin` (`listOf coercibleToStr`): Optional. + Append `/bin` + of each element in the list + to PATH. + Defaults to `[ ]`. + + - `rpath` (`listOf coercibleToStr`): Optional. + Append `/lib` and `/lib64` + of each element in the list + to LD_LIBRARY_PATH. + Defaults to `[ ]`. + + - `source` (`listOf coercibleToStr`): Optional. + Source (as in Bash's `source` command) + each element in the list. + Defaults to `[ ]`. + +Types specific to Crystal: + +- makeSearchPaths (`function { ... } -> package`): + + - `crystalLib` (`listOf coercibleToStr`): Optional. + Append `/lib` + of each element in the list + to CRYSTAL_LIBRARY_PATH. + Defaults to `[ ]`. + +Types specific to Java: + +- makeSearchPaths (`function { ... } -> package`): + + - `javaClass` (`listOf coercibleToStr`): Optional. + Append each element in the list + to CLASSPATH. + Defaults to `[ ]`. + +Types specific to Kubernetes: + +- makeSearchPaths (`function { ... } -> package`): + + - `kubeConfig` (`listOf coercibleToStr`): Optional. + Append each element in the list + to [KUBECONFIG](https://kubernetes.io/docs/concepts/configuration/). + Defaults to `[ ]`. + +Types specific to pkg-config: + +- makeSearchPaths (`function { ... } -> package`): + + - `pkgConfig` (`listOf coercibleToStr`): Optional. + Append `/lib/pkgconfig` + of each element in the list + to PKG_CONFIG_PATH. + Defaults to `[ ]`. + +Types specific to OCaml: + +- makeSearchPaths (`function { ... } -> package`): + + - `ocamlBin` (`listOf coercibleToStr`): Optional. + Append `/bin` + of each element in the list + to PATH. + Defaults to `[ ]`. + + - `ocamlLib` (`listOf coercibleToStr`): Optional. + Append `/` + of each element in the list + to OCAMLPATH. + Defaults to `[ ]`. + + - `ocamlStublib` (`listOf coercibleToStr`): Optional. + Append `/stublib` + of each element in the list + to CAML_LD_LIBRARY_PATH. + Defaults to `[ ]` + +Types specific to Python: + +- makeSearchPaths (`function { ... } -> package`): + + - `pythonMypy` (`listOf coercibleToStr`): Optional. + Append `/` + of each element in the list + to MYPYPATH. + Defaults to `[ ]`. + + - `pythonMypy38` (`listOf coercibleToStr`): Optional. + Append `/lib/python3.8/site-packages` + of each element in the list + to MYPYPATH. + Defaults to `[ ]`. + + - `pythonMypy39` (`listOf coercibleToStr`): Optional. + Append `/lib/python3.9/site-packages` + of each element in the list + to MYPYPATH. + Defaults to `[ ]`. + + - `pythonMypy310` (`listOf coercibleToStr`): Optional. + Append `/lib/python3.10/site-packages` + of each element in the list + to MYPYPATH. + Defaults to `[ ]`. + + - `pythonMypy311` (`listOf coercibleToStr`): Optional. + Append `/lib/python3.11/site-packages` + of each element in the list + to MYPYPATH. + Defaults to `[ ]`. + + - `pythonPackage` (`listOf coercibleToStr`): Optional. + Append `/` + of each element in the list + to PYTHONPATH. + Defaults to `[ ]`. + + - `pythonPackage38` (`listOf coercibleToStr`): Optional. + Append `/lib/python3.8/site-packages` + of each element in the list + to PYTHONPATH. + Defaults to `[ ]`. + + - `pythonPackage39` (`listOf coercibleToStr`): Optional. + Append `/lib/python3.9/site-packages` + of each element in the list + to PYTHONPATH. + Defaults to `[ ]`. + + - `pythonPackage310` (`listOf coercibleToStr`): Optional. + Append `/lib/python3.10/site-packages` + of each element in the list + to PYTHONPATH. + Defaults to `[ ]`. + + - `pythonPackage311` (`listOf coercibleToStr`): Optional. + Append `/lib/python3.11/site-packages` + of each element in the list + to PYTHONPATH. + Defaults to `[ ]`. + +Types specific to Node.js: + +- makeSearchPaths (`function { ... } -> package`): + + - `nodeBin` (`listOf coercibleToStr`): Optional. + Append `/.bin` + of each element in the list + to PATH. + Defaults to `[ ]`. + + - `nodeModule` (`listOf coercibleToStr`): Optional. + Append `/` + of each element in the list + to NODE_PATH. + Defaults to `[ ]`. + +Types specific to Ruby: + +- makeSearchPaths (`function { ... } -> package`): + + - `rubyBin` (`listOf coercibleToStr`): Optional. + Append `/bin` + of each element in the list + to PATH. + Defaults to `[ ]`. + + - `rubyGemPath` (`listOf coercibleToStr`): Optional. + Append `/` + of each element in the list + to GEM_PATH. + Defaults to `[ ]`. + +Types for non covered cases: + +- makeSearchPaths (`function { ... } -> package`): + + - `export` (`listOf (tuple [ str coercibleToStr str ])`): Optional. + Export (as in Bash's `export` command) + each tuple in the list. + Defaults to `[ ]`. + + Tuples elements are: + + - Name of the environment variable to export. + - Base package to export from. + - Relative path with respect to the package that should be appended. + + Example: + + ```bash + # /path/to/my/project/makes/example/template + echo "${@}" + ``` + + ```nix + # /path/to/my/project/makes/example/main.nix + makeSearchPaths { + source = [ + [ ./template "a" "b" "c" ] + # add more as you need ... + ]; + export = [ + [ "PATH" inputs.nixpkgs.bash "/bin"] + [ "CPATH" inputs.nixpkgs.glib.dev "/include/glib-2.0"] + # add more as you need ... + ]; + } + ``` + + Is equivalent to: + + ```bash + export PATH="/nix/store/...-bash/bin${PATH:+:}${PATH:-}" + export CPATH="/nix/store/...-glib-dev/include/glib-2.0${CPATH:+:}${CPATH:-}" + + if test -e "/nix/store/...-template/template" + then source "/nix/store/...-template/template" '1' '2' '3' + else source "/nix/store/...-template" '1' '2' '3' + fi + ``` + +Example: + +```nix +{ makeSearchPaths +, ... +}: +makeSearchPaths { + bin = [ inputs.nixpkgs.git ]; +} +``` + +## makeDerivation + +Perform a build step in an **isolated** environment: + +- External environment variables are not visible by the builder script. + This means you **can't** use secrets here. +- Search Paths as in `makeSearchPaths` are completely empty. +- The `HOME` environment variable is set to `/homeless-shelter`. +- Only [GNU coreutils][gnu_coreutils] commands (cat, echo, ls, ...) + are present by default. +- An environment variable called `out` is present + and represents the derivation's output. + The derivation **must** produce an output, + may be a file, or a directory. +- Convenience bash functions are exported: + + - `echo_stderr`: Like `echo` but to standard error. + - `debug`: Like `echo_stderr` but with a `[DEBUG]` prefix. + - `info`: Like `echo_stderr` but with a `[INFO]` prefix. + - `warn`: Like `echo_stderr` but with a `[WARNING]` prefix. + - `error`: Like `echo_stderr` but with a `[ERROR]` prefix. + Returns exit code 1 to signal failure. + - `critical`: Like `echo_stderr` but with a `[CRITICAL]` prefix. + Exits immediately with exit code 1, aborting the entire execution. + - `copy`: Like `cp` but making paths writeable after copying them. + - `require_env_var`: `error`s when the specified env var is not set, + or set to an empty value. + + ```bash + require_env_var USERNAME + ``` + +- After the build, for all paths in `$out`: + - User and group ownership are removed + - Last-modified timestamps are reset to `1970-01-01T00:00:00+00:00`. + +Types: + +- makeDerivation (`function { ... } -> package`): + - builder (`either str package`): + A Bash script that performs the build step. + - env (`attrsOf str`): Optional. + Environment variables that will be propagated to the `builder`. + Variable names must start with `env`. + Defaults to `{ }`. + - local (`bool`): Optional. + Should we always build locally this step? + Thus effectively ignoring any configured binary caches. + Defaults to `false`. + - name (`str`): + Custom name to assign to the build step, be creative, it helps in debugging. + - searchPaths (`asIn makeSearchPaths`): Optional. + Arguments here will be passed as-is to `makeSearchPaths`. + Defaults to `makeSearchPaths`'s defaults. + +Example: + +```nix +# /path/to/my/project/makes/example/main.nix +{ inputs +, makeDerivation +, ... +}: +makeDerivation { + env = { + envVersion = "1.0"; + }; + builder = '' + debug Version is $envVersion + info Running tree command on $PWD + mkdir dir + touch dir/file + tree dir > $out + ''; + name = "example"; + searchPaths = { + bin = [ inputs.nixpkgs.tree ]; + }; +} +``` + +```bash +$ m . /example + + [DEBUG] Version is 1.0 + [INFO] Running tree command on /tmp/nix-build-example.drv-0 + /nix/store/30hg7hzn6d3zmfva1bl4zispqilbh3nm-example + +$ cat /nix/store/30hg7hzn6d3zmfva1bl4zispqilbh3nm-example + dir + `-- file + + 0 directories, 1 file +``` + +## makeTemplate + +Replace placeholders with the specified values +in a file of any format. + +Types: + +- makeTemplate (`function { ... } -> package`): + - local (`bool`): Optional. + Should we always build locally this step? + Thus effectively ignoring any configured binary caches. + Defaults to `true`. + - name (`str`): + Custom name to assign to the build step, be creative, it helps in debugging. + - replace (`attrsOf strLike`): Optional. + Placeholders will be replaced in the script with their respective value. + Variable names must start with `__arg`, end with `__` + and have at least 6 characters long. + Defaults to `{ }`. + - template (`either str package`): + A string, file, output or package + in which placeholders will be replaced. + +Example: + +```nix +# /path/to/my/project/makes/example/main.nix +{ inputs +, makeTemplate +, ... +}: +makeTemplate { + name = "example"; + replace = { + __argBash__ = inputs.nixpkgs.bash; + __argVersion__ = "1.0"; + }; + template = '' + Bash is: __argBash__ + Version is: __argVersion__ + ''; +} +``` + +```bash +$ m . /example + + Bash is: /nix/store/kxj6cblcsd1qcbbxlmbswwrn89zcmgd6-bash-4.4-p23 + Version is: 1.0 +``` + +## makeScript + +Wrap a Bash script +that runs in a **almost-isolated** environment. + +- The file system is **not** isolated, the script runs in user-space. +- External environment variables are visible by the script. + You can use this to propagate secrets. +- Search Paths as in `makeSearchPaths` are completely empty. +- The `HOME_IMPURE` environment variable is set to the user's home directory. +- The `HOME` environment variable is set to a temporary directory. +- Only [GNU coreutils][gnu_coreutils] commands (cat, echo, ls, ...) + are present by default. +- An environment variable called `STATE` points to a directory + that can be used to store the script's state (if any). + That state can be optionally persisted. + That state can be optionally shared across repositories. +- Convenience bash functions are exported: + + - `running_in_ci_cd_provider`: + Detects if we are running on the CI/CD provider (gitlab/github/etc). + + ```bash + if running_in_ci_cd_provider; then + # ci/cd logic + else + # non ci/cd logic + fi + ``` + + - `prompt_user_for_confirmation`: + Warns the user about a possibly destructive action + that will be executed soon + and aborts if the user does not confirm aproppriately. + + This function assumes a positive answer + when running on the CI/CD provider + because there is no human interaction. + + - `prompt_user_for_input`: + Ask the user to type information + or optionally use a default value by pressing ENTER. + + This function assumes the default value + when running on the CI/CD provider + because there is no human interaction. + + ```bash + user_supplied_input="$(prompt_user_for_input "default123123")" + + info Supplied input: "${user_supplied_input}" + ``` + +- After the build, the script is executed. + +Types: + +- makeScript (`function { ... } -> package`): + - entrypoint (`either str package`): + A Bash script that performs the build step. + - name (`str`): + Custom name to assign to the build step, be creative, it helps in debugging. + - replace (`attrsOf strLike`): Optional. + Placeholders will be replaced in the script with their respective value. + Variable names must start with `__arg`, end with `__` + and have at least 6 characters long. + Defaults to `{ }`. + - searchPaths (`asIn makeSearchPaths`): Optional. + Arguments here will be passed as-is to `makeSearchPaths`. + Defaults to `makeSearchPaths`'s defaults. + - persistState (`bool`): Optional. + If true, state will _not_ be cleared before each script run. + Defaults to `false`. + - globalState (`bool`): Optional. + If true, script state will be written to `globalStateDir` and + to `projectStateDir` otherwise. + Defaults to `false`, if `projectStateDir` is specified or derived. + Note: + - It is implicitly `true`, if `projectStateDir == globalStateDir`. + - `projectStateDir == globalStateDir` is the default if + `projectIdentifier` is not configured. + - Hence, generally enable project local state by + - either setting `projectIdentifier` + - or `projectStateDir` different from `globalStateDir`. + +Example: + +```nix + +# /path/to/my/project/makes/example/main.nix +{ inputs +, makeScript +, ... +}: +makeScript { + replace = { + __argVersion__ = "1.0"; + }; + entrypoint = '' + debug Version is __argVersion__ + info pwd is $PWD + info Running tree command on $STATE + mkdir $STATE/dir + touch $STATE/dir/file + tree $STATE + ''; + name = "example"; + searchPaths = { + bin = [ inputs.nixpkgs.tree ]; + }; +} +``` + +```bash +$ m . /example + + [DEBUG] Version is 1.0 + [INFO] pwd is /data/github/fluidattacks/makes + [INFO] Running tree command on /home/user/.makes/state/example + /home/user/.makes/state/example + └── dir + └── file + + 1 directory, 1 file +``` + +## projectPath + +Copy a path from the current Makes project +being evaluated to the Nix store +in the **most** pure and reproducible way possible. + +Types: + +- projectPath (`function str -> package`): + + - (`str`): + Absolute path, assumming the repository is located at `"/"`. + +Example: + +```nix +# Consider the following path within the repository: /src/nix + +# /path/to/my/project/makes/example/main.nix +{ makeScript +, projectPath +, ... +}: +makeScript { + replace = { + __argPath__ = projectPath "/src/nix"; + }; + entrypoint = '' + info Path is: __argPath__ + info Path contents are: + ls __argPath__ + ''; + name = "example"; +} +``` + +```bash +$ m . /example + + [INFO] Path is: + [INFO] Path contents are: + packages.nix sources.json sources.nix +``` + +[GNU Coreutils][gnu_coreutils] diff --git a/docs/src/api/extensions/git.md b/docs/src/api/extensions/git.md new file mode 100644 index 000000000..c6c97797c --- /dev/null +++ b/docs/src/api/extensions/git.md @@ -0,0 +1,71 @@ +## libGit + +A small template for doing git kung-fu. + +Types: + +- libGit (`package`): + A package that can be sourced to setup functions in the current scope. + The list of available functions is documented below: + + - `is_git_repository`: + Return 0 if the provided path is a git repository. + + ```bash + if is_git_repository /path/to/anywhere; then + # custom logic + fi + ``` + + - `require_git_repository`: + Stops the execution + if the provided path is not a git repository. + + ```bash + require_git_repository /path/to/anywhere + ``` + + - `get_abbrev_rev`: + If available, returns an abbreviated name for the provided revision. + Otherwise returns the revision unchanged. + + ```bash + # Would return main, trunk, develop, etc + get_abbrev_rev /path/to/anywhere HEAD + ``` + + - `get_commit_from_rev`: + If available, returns the full commit of the provided revision. + Otherwise returns an error. + + ```bash + # Would return the full commit (e026a413...) + get_commit_from_rev /path/to/anywhere HEAD + ``` + +Example: + +```nix +# /path/to/my/project/makes/example/main.nix +{ libGit +, makeScript +, ... +}: +makeScript { + entrypoint = '' + require_git_repository /some-path-that-do-not-exists + + echo other business logic goes here ... + ''; + name = "example"; + searchPaths = { + source = [ libGit ]; + }; +} +``` + +```bash +$ m . /example + + [CRITICAL] We require a git repository, but this one is not: /some-path-that-do-not-exists +``` diff --git a/docs/src/api/extensions/index.md b/docs/src/api/extensions/index.md new file mode 100644 index 000000000..8da2aea50 --- /dev/null +++ b/docs/src/api/extensions/index.md @@ -0,0 +1,68 @@ +# Introduction + +You can create custom workflows +not covered by the builtin `makes.nix` configuration options. + +In order to do this: + +Locate in the root of your project: + +`$ cd /path/to/my/project` + +Create a directory structure. In this case: `makes/example`. + +`$ mkdir -p makes/example` + +We will place in this directory +all the source code +for the custom workflow called `example`. + +Create a `main.nix` file inside `makes/example`. + +Our goal is to create a bash script that prints `Hello from makes!`, +so we are going to write the following function: + +```nix +# /path/to/my/project/makes/example/main.nix +{ makeScript +, ... +}: +makeScript { + entrypoint = "echo Hello from Makes!"; + name = "hello-world"; +} +``` + +Now run makes! + +- List all available outputs: `$ m .` + + ``` + Outputs list for project: /path/to/my/project + /example + ``` + +- Build and run the output: `$ m . /example` + + ``` + Hello from Makes! + ``` + +Makes will automatically recognize as outputs all `main.nix` files +under the `makes/` directory in the root of the project. +This "magic" `makes/` directory can be configured via the +`extendingMakesDirs` option. + +You can create any directory structure you want. +Output names will me mapped in an intuitive way: + +| `main.nix` position | Output name | Invocation command | +| -------------------------------------------------- | -------------------------- | ---------------------- | +| `/path/to/my/project/makes/main.nix` | `outputs."/"` | `$ m . /` | +| `/path/to/my/project/makes/example/main.nix` | `outputs."/example"` | `$ m . /example` | +| `/path/to/my/project/makes/other/example/main.nix` | `outputs."/other/example"` | `$ m . /other/example` | + +Makes offers you a few building blocks +for you to reuse. + +Let's start from the basics. diff --git a/docs/src/api/extensions/node.js.md b/docs/src/api/extensions/node.js.md new file mode 100644 index 000000000..ef9f363ed --- /dev/null +++ b/docs/src/api/extensions/node.js.md @@ -0,0 +1,213 @@ +## makeNodeJsVersion + +Get a specific Node.js version interpreter. + +Types: + +- makeNodeJsVersion (`function str -> package`): + + - (`enum [ "14" "16" "18" ]`): + Node.js version to use. + +Example: + +```nix +# /path/to/my/project/makes/example/main.nix +{ makeNodeJsVersion +, makeScript +, ... +}: +makeScript { + entrypoint = '' + node --version + ''; + name = "example"; + searchPaths = { + bin = [ (makeNodeJsVersion "16") ]; + }; +} +``` + +```bash +$ m . /example + + v16.2.0 +``` + +## makeNodeJsModules + +:warning: This function is only available on Linux at the moment. + +Cook the `node_modules` directory +for the given Node.js project. + +Types: + +- makeNodeJsModules (`function { ... } -> package`): + + - name (`str`): + Custom name to assign to the build step, be creative, it helps in debugging. + - nodeJsVersion (`enum [ "14" "16" "18" ]`): + Node.js version to use. + - packageJson (`package`): + Path to the `package.json` of your project. + - packageLockJson (`package`): + Path to the `package-lock.json` of your project. + - searchPaths (`asIn makeSearchPaths`): Optional. + Arguments here will be passed as-is to `makeSearchPaths`. + Defaults to `makeSearchPaths`'s defaults. + - shouldIgnoreScripts (`bool`): Optional. + Enable to propagate the `--ignore-scripts true` flag to npm. + Defaults to `false`. + +Example: + +```json +# /path/to/my/project/makes/example/package.json +{ + "dependencies": { + "hello-world-npm": "*" + } +} +``` + +```json +# /path/to/my/project/makes/example/package-lock.json +{ + "requires": true, + "lockfileVersion": 1, + "dependencies": { + "hello-world-npm": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/hello-world-npm/-/hello-world-npm-1.1.1.tgz", + "integrity": "sha1-JQgw7wAItDftk+a+WZk0ua0Lkwg=" + } + } +} +``` + +```nix +# /path/to/my/project/makes/example/main.nix +{ makeNodeJsModules +, makeScript +, projectPath +, ... +}: +let + hello = makeNodeJsModules { + name = "hello-world-npm"; + nodeJsVersion = "16"; + packageJson = + projectPath "/path/to/my/project/makes/example/package.json"; + packageLockJson = + projectPath "/path/to/my/project/makes/example/package-lock.json"; + }; +in +makeScript { + replace = { + __argHello__ = hello; + }; + entrypoint = '' + ls __argHello__ + ''; + name = "example"; +} +``` + +```bash +$ m . /example + + hello-world-npm +``` + +## makeNodeJsEnvironment + +:warning: This function is only available on Linux at the moment. + +Setup a `makeNodeJsModules` in the environment +using `makeSearchPaths`. +It appends: + +- `node` to `PATH`. +- `node_modules/.bin` to `PATH`. +- `node_modules` to [NODE_PATH](https://nodejs.org/api/modules.html). + +Pre-requisites: [Generating a package-lock.json](#makenodejslock) + +Types: + +- makeNodeJsEnvironment (`function { ... } -> package`): + + - name (`str`): + Custom name to assign to the build step, be creative, it helps in debugging. + - nodeJsVersion (`enum [ "14" "16" "18" ]`): + Node.js version to use. + - packageJson (`package`): + Path to the `package.json` of your project. + - packageLockJson (`package`): + Path to the `package-lock.json` of your project. + - searchPaths (`asIn makeSearchPaths`): Optional. + Arguments here will be passed as-is to `makeSearchPaths`. + Defaults to `makeSearchPaths`'s defaults. + - shouldIgnoreScripts (`bool`): Optional. + Enable to propagate the `--ignore-scripts true` flag to npm. + Defaults to `false`. + +Example: + +```json +# /path/to/my/project/makes/example/package.json +{ + "dependencies": { + "hello-world-npm": "*" + } +} +``` + +```json +# /path/to/my/project/makes/example/package-lock.json +{ + "requires": true, + "lockfileVersion": 1, + "dependencies": { + "hello-world-npm": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/hello-world-npm/-/hello-world-npm-1.1.1.tgz", + "integrity": "sha1-JQgw7wAItDftk+a+WZk0ua0Lkwg=" + } + } +} +``` + +```nix +# /path/to/my/project/makes/example/main.nix +{ makeNodeJsEnvironment +, makeScript +, ... +}: +let + hello = makeNodeJsEnvironment { + name = "hello-world-npm"; + nodeJsVersion = "16"; + packageJson = + projectPath "/path/to/my/project/makes/example/package.json"; + packageLockJson = + projectPath "/path/to/my/project/makes/example/package-lock.json"; + }; +in +makeScript { + entrypoint = '' + hello-world-npm + ''; + name = "example"; + searchPaths = { + source = [ hello ]; + }; +} +``` + +```bash +$ m . /example + + Hello World NPM +``` diff --git a/docs/src/api/extensions/others.md b/docs/src/api/extensions/others.md new file mode 100644 index 000000000..aa9c0bdf9 --- /dev/null +++ b/docs/src/api/extensions/others.md @@ -0,0 +1,220 @@ +## chunks + +Split a given list into N chunks +for workload distributed parallelization. + +Types: + +- chunks (`function list, ints.positive -> listOf (listOf Any)`): + + - (`list`): + List to split into chunks. + - (`ints.positive`): + Number of chunks to create from list. + +Example: + +```nix +{ + chunks, + inputs, + makeDerivation, + makeDerivationParallel, +}: let +numbers = [0 1 2 3 4 5 6 7 8 9]; +myChunks = chunks numbers 3; # [[0 1 2 3] [4 5 6] [7 8 9]] + +buildNumber = n: makeDerivation { + name = "build-number-${n}"; + env.envNumber = n; + builder = '' + echo "$envNumber" + touch "$out" + ''; +}; +in + makeDerivationParallel { + dependencies = builtins.map buildNumber (inputs.nixpkgs.lib.lists.elemAt myChunks 0); + name = "build-numbers-0"; + } +``` + +## calculateCvss3 + +Calculate [CVSS3](https://www.first.org/cvss/v3.0/specification-document) +score and severity for a +[CVSS3 Vector String](https://www.first.org/cvss/v3.0/specification-document#Vector-String). + +Types: + +- calculateCvss3 (`function str -> package`): + + - (`str`): + CVSS3 Vector String + to calculate. + +Example: + +```nix +# /path/to/my/project/makes/example/main.nix +{ makeScript +, calculateCvss3 +, ... +}: +makeScript { + replace = { + __argCalculate__ = calculateCvss3 + "CVSS:3.0/S:C/C:H/I:H/A:N/AV:P/AC:H/PR:H/UI:R/E:H/RL:O/RC:R/CR:H/IR:X/AR:X/MAC:H/MPR:X/MUI:X/MC:L/MA:X"; + }; + entrypoint = '' + cat "__argCalculate__" + ''; + name = "example"; +} +``` + +```bash +$ m . /example + + {"score": {"base": 6.5, "temporal": 6.0, "environmental": 5.3}, "severity": {"base": "Medium", "temporal": "Medium", "environmental": "Medium"}} +``` + +## makeSslCertificate + +Self sign certificates +by using the `openssl req` command, +then using `openssl x509` +to print out the certificate +in text form. + +Types: + +- makeSslCertificate (`function { ... } -> package`): + + - days (`ints.positive`): Optional. + Ammount of days to certify the certificate for. + Defaults to `30`. + - keyType (`str`): Optional. + Defines the key type for the certificate + (option used for the `-newkey` option on the `req` command). + It uses the form `rsa:nbits`, where `nbits` is the number of bits. + Defaults to `rsa:4096`. + - name (`str`): + Custom name to assign to the build step, be creative, it helps in debugging. + - options (`listOf (listOf str)`): + Contains a list of options to create the certificate with your own needs. + Here you can use the same options used with `openssl req`. + +Example: + +```nix +# /path/to/my/project/makes/example/main.nix +{ makeScript +, makeSslCertificate +, ... +}: +let + sslCertificate = makeSslCertificate { + name = "name-example"; + options = [ + [ "-subj" "/CN=localhost" ] + ]; + }; +in +makeScript { + replace = { + __argSslCertificate__ = sslCertificate; + }; + entrypoint = '' + cat "__argSslCertificate__" + ''; + name = "example"; +} + +``` + +```bash +$ m . /example + + -----BEGIN PRIVATE KEY----- + MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDQ/tFdIW3kL1IJ + hjD11ZTTDvMXlO+Zm3Oc3Z67Kb9llNpdgyDXBXyFriAfDsDAw/Hrp7zSqzNMT5Qh + fc1OFhM2ICuPaFCQONKsOulo3LhdjXfSDuvu1k0XMF0cDOVwhQkxYdsE9/jZQUSi + CB4I2A8LLnkMvZ+ANJIFzjkxey2A+v3KMeE5aw2PLqj8H+jAuxM56fCgFrXhhmPh + U6HNlhf+dvaV/PHjpr66HJf8gF/DwhzQ+ppYbVsBnuvzmiTz/dix9wu7m3/RxVIM + OwEPcZU2VCXT2MCtXKd6H+h8vEdx1xrLYRTWUhrOnNgNrblXhlpo0frI6XloujR3 + o5x18/GjCht6gx3D/ze+g6bhgGVUxMIMuM5uCLyOxCG/M23WaAZVOgseqnggCVP7 + MA/c7sd+cIWfS5Yi04G0vXkiQDUMOhRIZM2CFKr6Lyi6hdU2bUkx9gWSQWPMcwko + kFRkv1UpzWMfD8nMphuWwMJZckNDrmUb4kK8bfum/Q+9/EYNREEpDxz6OUNezjbq + g5r8lFNcfmAySZXFGSZdANS+u1CpcyWzMtgUIvjtANeqjJw+zOMqQBBeSROpgusY + z9N8G3ZPkArIKTpKRpPdwfIPCALd5ZLrdZAJMuPTHBFGRn/oxWg/zHNYBkLTiPAJ + R9V8mz4Q59WFoF92vNmPVp1bcBtqjQIDAQABAoICAETZjXNWzfL8O9RzZrG1+N9G + 74J3SC+cbIvi3qXd7PW0AfQIfMsZPZp0cJSKsalPY+U0Txo/2BhtpukZnob48r9D + /dWykcfRUGX4ymgHPp1jO3PyAnueEatv/Vx+Sx+0VAD2scaDQnGf9NJERlC3jM0s + NCikV2VO7EQJWgIZCDaTeQQhRoc54y+mOXlFsdG8T2smzGSQ1r5DHahfetBVf/YO + jtF+kNlkVzTMsq02RVYiHogh5rL246I3Dpgj0cSnfbmzHyRg58zdalgpIAJMctGB + Cy0tHNx/x5SN6nMdO5QfPu59PvYT+kzSksJ+1q4kzqf1dN63O43qudooBiU9hf6N + RrocVXwr/vXyo78gOKN706guxf5VP3+4ldeV5AP2tCSEsI0D2H/Mbt3PlLce6zxc + BMb6lpM6XddMXkqU993ewhMqRMvvSuqNXuHp6dYmt76v0yL3GwaB3rqt1BtwPwOx + TNmoP4wcoAjKL1Q66cIFW9oZT+XuxZFn57Ch6hrxNLKzEOgtbGtuCNuCQoDVBWX5 + CLj2Oh+rK9v0Zmz4NNNDsmY3m3ViMQz4i52PuChtm3E/2dR1GsoQsy8Js3IWC9g+ + Kr7cGyL3KcOVc7snx6d6CRqKaZTsiDOX5GlQbHb3KKUAVJlGB37rXITy/yFnPffz + Rv91dVb/RFucmagBDUABAoIBAQD5A3+sHjQYgQI10ceiqEDqzqyjAog4Jg91Qt8M + qII8fUZC2yUF2kyHYeokEr4LXiYSS5GvNJ0Fm8BF4/ShuRECkU+tG73Y8JMChjfD + CU89d0G4X2I6AZShOAaWd4ypW5CsoVC7fa5Jzbd/C8RucXDJlZQlpl0Dr4VEs53Y + +a6uSmw4EPMTTrCLH2bqWXLuW3reaZM38QrbLhk5kZRTYvo2YOdmfscFop4TbEJF + qpOA0E2N5iWUnU4K6fVcBKIycsz2Ao9D1Jk1NzZGxvcvc0YxjoBSyqLZOZ9Z3Wh7 + YkfyUs4TuaEHP6/JxGBWTs7jmKdigeFfsNYRS8BFdSs3jsgBAoIBAQDW2+TkJBm2 + oXbTIuNSoYxdJocdScktLAfVH+YzrSQ78dbSs5hcVwS0WugD30Fr1n0cZ7EttbZE + FF5ZxzqHVV0mrMH64OiuhVTW6bXzlJ+V6bs4Sq/fL/iiEIoNm5D2GNrWbcYEIJ8h + Fj6QOyp8VnkAfLmYTb5ohlNQVZLr8IMpeTF4vQUCkP9umtn0S+lp8ZXNy6yTNSay + mqxbZ0pju79nk+W99TapM4FNXLoHjtkqBVu4XYlS4qX8gd1plMzZdSvUW8fWjUhR + BvbW+dqxCuryBjbHkB7rl3dSFFvl8I3JPy4fiIkHln3OEe0Cpas/IrTY+xshA3kt + kMNE+3SO50KNAoIBAQDF0Wi4dBoQqVP3K1r7tcw0fNEagmVyrZG0JtaI+MjVgvOx + IuSLfLs1Baz60UTWRQnbmNr4I8Tl8rBRFWF+pEWGE6gHLjWoRJ2U8MkVkKy5eKbl + 8ChZSm4nkRlyqTA+TjZlXZWEDLjLerheHhwDXO0rxz80la/owKQPSt2Hw/poDUlh + VN21pdqL+vtICp1KC7RVQeupEjz8l+eEG0mI4OVDE8JgYzB6IpCPf346V+Lr/w7N + Plr2b+zSsL+xRSERELAQc0IasaawZtcgbOlrcZj+v2Tj4IR0KtmTi1d4RUBAmlWJ + x/rLhmWA1RdvGRY0Kk427FT9Lr8waEwrIYSekzgBAoIBACEDjLoZafIMAUwT8kYC + GKU/hEdVzRmpyFJRIngSRJ0JXe7mNaUKoehsh3YA2faN8I9qx2i0oRr43j6BRFcD + INsOdIfuAxK93flf09tnnNXWIjRWFYv/vP55+Bx7KN0HmKiWGXUM5iaZWmejD7Yn + O1R91a63U2iQK0EOxRKH1D+NJbLdqGVqjjUaih7lgyoKOvByOUQtSJLs/UrWJjII + 6TIrIYP8p7d7+IRAmT0MEAZK6Hr9tFoOBV81PSY5/Pf07xUkPSKUduYsYcVKgvXt + LYiet9AWLwoYLfdotW4xdjfUA2xI+HU4BICjdH2RoyyCUrN8cgCyne4IblitIo3K + rwkCggEBAN0xdTlbZEI/r1O3iDNJXcXJg3HUMj78pz7c32ROMS2iwsQTyj+IHui8 + 0J2FOdZ8TUlgoBfi3C1Y2NyNdyAJ3jiHnCrQz/sqTRYGds+aALfw1YZZuonUXAwc + OxCZcMowzTvx5iCcaCY9jsdrr4TYGWSf2wmzSD87EYqNKLTd4asOCILatTWMw0AR + xBHKugWHSokf9SNzirqxSNeqjjepMTA95LRiijKQAu9yhj0Zs35EUIu88KA5PZ4q + 0+URRTIuCtyjKBFC5qBhvbWzKe46hSy6OPyJFPgyo4OCC0NkesLQKcJwfTckK8Ne + mSjLja2l8YqKkXqV6P3R6wVLMvCoCao= + -----END PRIVATE KEY----- + +``` + +## sublist + +Return a sublist of a given list using a starting and an ending index. + +Types: + +- sublist (`function list, ints.positive, ints.positive -> listOf Any`): + + - (`list`): + List to get sublist from. + - (`ints.positive`): + Starting list index. + - (`ints.positive`): + Ending list index. + +Example: + +```nix +{ + sublist, +}: let + list = [0 1 2 3 4 5 6 7 8 9]; + sublist = sublist list 3 5; # [3 4] +in { + inherit sublist; +} +``` diff --git a/docs/src/api/extensions/patchers.md b/docs/src/api/extensions/patchers.md new file mode 100644 index 000000000..187c0a1a6 --- /dev/null +++ b/docs/src/api/extensions/patchers.md @@ -0,0 +1,62 @@ +## pathShebangs + +Replace common [shebangs](https://bash.cyberciti.biz/guide/Shebang) +for their Nix equivalent. + +For example: + +- `#! /bin/env xxx` -> `/nix/store/..-name/bin/xxx` +- `#! /usr/bin/env xxx` -> `/nix/store/..-name/bin/xxx` +- `#! /path/to/my/xxx` -> `/nix/store/..-name/bin/xxx` + +Types: + +- pathShebangs (`package`): + When sourced, + it exports a Bash function called `patch_shebangs` + into the evaluation context. + This function receives one or more files or directories as arguments + and replace shebangs of the executable files in-place. + Note that only shebangs that resolve to executables in the `"${PATH}"` + (a.k.a. `searchPaths.bin`) will be taken into account. + +Examples: + +```nix +# /path/to/my/project/makes/example/main.nix +{ __nixpkgs__ +, makeDerivation +, patchShebangs +, ... +}: +makeDerivation { + env = { + envFile = builtins.toFile "my_file.sh" '' + #! /usr/bin/env bash + + echo Hello! + ''; + }; + builder = '' + copy $envFile $out + + chmod +x $out + patch_shebangs $out + + cat $out + ''; + name = "example"; + searchPaths = { + bin = [ __nixpkgs__.bash ]; # Propagate bash so `patch_shebangs` "sees" it + source = [ patchShebangs ]; + }; +} +``` + +```bash +$ m . /example + + #! /nix/store/dpjnjrqbgbm8a5wvi1hya01vd8wyvsq4-bash-4.4-p23/bin/bash + + echo Hello! +``` diff --git a/docs/src/api/extensions/python.md b/docs/src/api/extensions/python.md new file mode 100644 index 000000000..7f2c9d33c --- /dev/null +++ b/docs/src/api/extensions/python.md @@ -0,0 +1,164 @@ +## makePythonVersion + +Get a specific Python interpreter. + +Types: + +- makePythonVersion (`function str -> package`): + + - (`enum ["3.8" "3.9" "3.10" "3.11"]`): + Python version of the interpreter to return. + +Example: + +```nix +# /path/to/my/project/makes/example/main.nix +{ makePythonVersion +, makeScript +, ... +}: +makeScript { + entrypoint = '' + python --version + ''; + name = "example"; + searchPaths = { + bin = [ (makePythonVersion "3.8") ]; + }; +} +``` + +```bash +$ m . /example + + Python 3.8.9 +``` + +## makePythonPypiEnvironment + +Create a virtual environment +where a provided set of Python packages +from [PyPi](https://pypi.org/) +is installed. + +Pre-requisites: [Generating a sourcesYaml](#makepythonlock) + +Types: + +- makePythonPypiEnvironment (`function { ... } -> package`): + + - name (`str`): + Custom name to assign to the build step, be creative, it helps in debugging. + - searchPathsBuild (`asIn makeSearchPaths`): Optional. + Arguments here will be passed as-is to `makeSearchPaths` + and used while installing the Python dependencies. + Defaults to `makeSearchPaths`'s defaults. + - searchPathsRuntime (`asIn makeSearchPaths`): Optional. + Arguments here will be passed as-is to `makeSearchPaths` + and propagated to the runtime environment. + Defaults to `makeSearchPaths`'s defaults. + - sourcesYaml (`package`): + `sources.yaml` file + computed as explained in the pre-requisites section. + + For building a few special packages you may need to boostrap + dependencies in the build environment. + The following flags are available for convenience: + + - withCython_0_29_24 (`bool`): Optional. + Bootstrap cython 0.29.24 to the environment + Defaults to `false`. + - withNumpy_1_24_0 (`bool`): Optional. + Bootstrap numpy 1.24.0 to the environment + Defaults to `false`. + - withSetuptools_57_4_0 (`bool`): Optional. + Bootstrap setuptools 57.4.0 to the environment + Defaults to `false`. + - withSetuptoolsScm_5_0_2 (`bool`) Optional. + Bootstrap setuptools-scm 5.0.2 to the environment + Defaults to `false`. + - withSetuptoolsScm_6_0_1 (`bool`) Optional. + Bootstrap setuptools-scm 6.0.1 to the environment + Defaults to `false`. + - withWheel_0_37_0 (`bool`): Optional. + Bootstrap wheel 0.37.0 to the environment + Defaults to `false`. + +Example: + +```nix +# /path/to/my/project/makes/example/main.nix +{ inputs +, makePythonPypiEnvironment +, projectPath +, ... +}: +makePythonPypiEnvironment { + name = "example"; + # If some packages require compilers to be built, + # you can provide them like this: + searchPathsBuild = { + bin = [ inputs.nixpkgs.gcc ]; + }; + # You can propagate packages to the runtime environment if needed, too + searchPathsRuntime = { + bin = [ inputs.nixpkgs.htop ]; + }; + sourcesYaml = projectPath "/makes/example/sources.yaml"; + # Other packages require a few bootstrapped dependencies, + # enable them like this: + withCython_0_29_24 = true; + withSetuptools_57_4_0 = true; + withSetuptoolsScm_6_0_1 = true; + withWheel_0_37_0 = true; +} +``` + +`sourcesYaml` is generated like this: + +```bash +$ cat /path/to/my/project/makes/example/dependencies.yaml + + Django: "3.2.6" + +$ m github:fluidattacks/makes@23.04 /utils/makePythonLock \ + 3.8 \ + /path/to/my/project/makes/example/dependencies.yaml \ + /path/to/my/project/makes/example/sources.yaml + + # ... + +$ cat /path/to/my/project/makes/example/sources.yaml + + closure: + asgiref: 3.4.1 + django: 3.2.6 + pytz: "2021.1" + sqlparse: 0.4.1 + links: + - name: Django-3.2.6-py3-none-any.whl + sha256: 04qzllkmyl0g2fgdab55r7hv3vqswfdv32p77cgjj3ma54sl34kz + url: https://pypi.org/packages/py3/D/Django/Django-3.2.6-py3-none-any.whl + - name: Django-3.2.6.tar.gz + sha256: 08p0gf1n548fjba76wspcj1jb3li6lr7xi87w2xq7hylr528azzj + url: https://pypi.org/packages/source/D/Django/Django-3.2.6.tar.gz + - name: pytz-2021.1-py2.py3-none-any.whl + sha256: 1607gl2x9290ks5sa6dvqw9dgg1kwdf9fj9xcb9jw19nfwzcw47b + url: https://pypi.org/packages/py2.py3/p/pytz/pytz-2021.1-py2.py3-none-any.whl + - name: pytz-2021.1.tar.gz + sha256: 1nn459q7zg20n75akxl3ljkykgw1ydc8nb05rx1y4f5zjh4ak943 + url: https://pypi.org/packages/source/p/pytz/pytz-2021.1.tar.gz + - name: sqlparse-0.4.1-py3-none-any.whl + sha256: 1l2f616scnhbx7nkzvwmiqvpjh97x11kz1v1bbqs3mnvk8vxwz01 + url: https://pypi.org/packages/py3/s/sqlparse/sqlparse-0.4.1-py3-none-any.whl + - name: sqlparse-0.4.1.tar.gz + sha256: 1s2l0jgi1v7rk7smzb99iamasaz22apfkczsphn3ci4wh8pgv48g + url: https://pypi.org/packages/source/s/sqlparse/sqlparse-0.4.1.tar.gz + - name: asgiref-3.4.1-py3-none-any.whl + sha256: 052j8715bw39iywciicgfg5hxnsgmyvv7cg7fdb1fvwfj2m43hgz + url: https://pypi.org/packages/py3/a/asgiref/asgiref-3.4.1-py3-none-any.whl + - name: asgiref-3.4.1.tar.gz + sha256: 1saqgpgbdvb8awzm0f0640j0im55hkrfzvcw683cgqw4ni3apwaf + url: https://pypi.org/packages/source/a/asgiref/asgiref-3.4.1.tar.gz + python: "3.8" +``` diff --git a/docs/src/api/extensions/ruby.md b/docs/src/api/extensions/ruby.md new file mode 100644 index 000000000..0ec6bbf43 --- /dev/null +++ b/docs/src/api/extensions/ruby.md @@ -0,0 +1,134 @@ +## makeRubyVersion + +Get a specific Ruby interpreter. + +Types: + +- makeRubyVersion (`function str -> package`): + + - (`enum [ "2.7" "3.0" "3.1" ]`): + Version of the Ruby interpreter. + +Example: + +```nix +# /path/to/my/project/makes/example/main.nix +{ makeRubyVersion +, makeScript +, ... +}: +makeScript { + entrypoint = '' + ruby --version + ''; + name = "example"; + searchPaths = { + bin = [ (makeRubyVersion "2.7") ]; + }; +} +``` + +```bash +$ m . /example + + ruby 2.6.8p205 (2021-07-07) [x86_64-linux] +``` + +## makeRubyGemsInstall + +Fetch and install the specified Ruby gems +from the [RubyGems][rubygems]. + +Pre-requisites: [Generating a sourcesYaml](#makerubylock) + +Types: + +- makeRubyGemsInstall (`function { ... } -> package`): + + - name (`str`): + Custom name to assign to the build step, be creative, it helps in debugging. + - ruby (`enum [ "2.7" "3.0" ]`): + Version of the Ruby interpreter. + - searchPaths (`asIn makeSearchPaths`): Optional. + Arguments here will be passed as-is to `makeSearchPaths`. + Defaults to `makeSearchPaths`'s defaults. + - sourcesYaml (`package`): + `sources.yaml` file + computed as explained in the pre-requisites section. + +Example: + +```nix +# /path/to/my/project/makes/example/main.nix +{ makeRubyGemsInstall +, ... +}: +makeRubyGemsInstall { + name = "example"; + ruby = "3.0"; + sourcesYaml = projectPath "/makes/example/sources.yaml"; +} +``` + +## makeRubyGemsEnvironment + +Create an environment where the specified Ruby gems +from [RubyGems][rubygems] +are available. + +Pre-requisites: [Generating a sourcesYaml](#makerubylock) + +Types: + +- makeRubyGemsEnvironment (`function { ... } -> package`): + + - name (`str`): + Custom name to assign to the build step, be creative, it helps in debugging. + - ruby (`enum [ "2.7" "3.0" ]`): + Version of the Ruby interpreter. + - searchPathsBuild (`asIn makeSearchPaths`): Optional. + Arguments here will be passed as-is to `makeSearchPaths` + and used while installing gems. + Defaults to `makeSearchPaths`'s defaults. + - searchPathsRuntime (`asIn makeSearchPaths`): Optional. + Arguments here will be passed as-is to `makeSearchPaths` + and propagated to the runtime environment. + Defaults to `makeSearchPaths`'s defaults. + - sourcesYaml (`package`): + `sources.yaml` file + computed as explained in the pre-requisites section. + +Example: + +```nix +# /path/to/my/project/makes/example/main.nix +{ inputs +, makeRubyGemsEnvironment +, makeScript +, ... +}: +let + env = makeRubyGemsEnvironment { + name = "example"; + ruby = "3.0"; + searchPathsBuild.bin = [ inputs.nixpkgs.gcc ]; + searchPathsRuntime.rpath = [ inputs.nixpkgs.gcc.cc.lib ]; + sourcesYaml = projectPath "/makes/example/sources.yaml"; + }; +in +makeScript { + entrypoint = '' + slimrb --version + ''; + name = "example"; + searchPaths.source = [ env ]; +} +``` + +```bash +$ m . /example + + Slim 4.1.0 +``` + +[rubygems]: https://rubygems.org/