pipeline-spec.yaml

raise-exception:
  pipeline:
    - run: raise_exception
      code: raise Exception()

failure-no-errors:
  pipeline:
    - run: success
      code: ""

success:
  pipeline:
  - run: success
    code: |
          from datapackage_pipelines.wrapper import ingest, spew
          parameters, datapackage, resources = ingest()
          spew(datapackage, [])

verbose-logs-with-sleep:
  pipeline:
  - run: code
    code: |
      from datapackage_pipelines.wrapper import ingest, spew
      import logging, itertools, time
      log_numbers = itertools.count()
      def log_line():
        logging.info('log line {}'.format(next(log_numbers)))
      log_line()
      time.sleep(.1)
      log_line()
      time.sleep(.1)
      log_line()
      time.sleep(.1)
      parameters, datapackage, resources = ingest()
      log_line()
      time.sleep(.1)
      log_line()
      time.sleep(.1)
      log_line()
      time.sleep(.1)
      spew(datapackage, [])
      log_line()
      time.sleep(.1)
      log_line()
      time.sleep(.1)
      log_line()

load-resource-progress-log:
  pipeline:
  - run: load_resource
    parameters:
      url: ../data/datapackage.json
      resource: my-spiffy-resource
      log-progress-rows: 2

custom-formatters:
  pipeline:
  - run: load_resource
    parameters:
      url: ../data/datapackage.json
      resource: my-spiffy-resource
  - run: duplicate
    parameters:
      source: my-spiffy-resource
      target-name: my-spiffy-xlsx-resource
      target-path: my-spiffy-resource.xlsx
  - run: dump.to_path
    parameters:
      out-path: custom_formatters
      force-format: false
      counters:
        resource-hash: ''
      file-formatters:
        xlsx: custom_formatters.xlsx_format.XLSXFormat
  - run: dump.to_zip
    parameters:
      out-file: custom_formatters/datapackage.zip
      force-format: false
      counters:
        resource-hash: ''
      file-formatters:
        xlsx: custom_formatters.xlsx_format.XLSXFormat

dataflows:
  pipeline:
  - run: load_resource
    parameters:
      url: ../data/datapackage.json
      resource: my-spiffy-resource
  - flow: test_flow
    parameters:
      attr: foo
  - run: dump_to_path
    parameters:
      out-path: test_flow_data