Some perf flags exposed via env vars:

* Disabled http2 due to http2 streaming in net/http2 performance issues golang/go#37373 * RCLONE_S3_UPLOAD_CONCURRENCY, RCLONE_S3_CHUNK_SIZE to improve parallelism and download efficiency Signed-off-by: mzardab <[email protected]>
moadz · Sep 1, 2023 · ff7411e · ff7411e
1 parent bfc4891
commit ff7411e
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 2 deletions.
diff --git a/resources/operations/rclone-bucket-replicate/job-template.yaml b/resources/operations/rclone-bucket-replicate/job-template.yaml
@@ -74,8 +74,8 @@ objects:
                 - ${SOURCE_ENDPOINT}:${SOURCE_BUCKET}
                 - ${TARGET_ENDPOINT}:${TARGET_BUCKET}
                 - '--rc'
-                - '-v'
                 - '--rc-enable-metrics'
                 - '--rc-addr=:10902'
+                - '--s3-disable-http2' # Needed due to https://github.com/golang/go/issues/37373
           restartPolicy: Never
 
diff --git a/resources/operations/rclone-bucket-replicate/rclone-config-template.yaml b/resources/operations/rclone-bucket-replicate/rclone-config-template.yaml
@@ -16,11 +16,15 @@ parameters:
   - name: RCLONE_STATS_INTERVAL
     value: '30s'
   - name: RCLONE_LOG_LEVEL
-    value: '1'
+    value: '0'
   - name: RCLONE_PARALLEL_TRANSFERS
     value: '4'
   - name: RCLONE_PARALLEL_FILE_CHECKERS
     value: '8'
+  - name: RCLONE_S3_UPLOAD_CONCURRENCY
+    value: '8'
+  - name: RCLONE_S3_CHUNK_SIZE
+    value: '64M'
 
 objects:
   - apiVersion: v1
@@ -42,3 +46,7 @@ objects:
       # Checkers do the equality checking of files during a sync
       RCLONE_CHECKERS: ${RCLONE_PARALLEL_FILE_CHECKERS}
 
+      # AWS Client Specific Parameters
+      RCLONE_S3_UPLOAD_CONCURRENCY: ${RCLONE_S3_UPLOAD_CONCURRENCY}
+      RCLONE_S3_CHUNK_SIZE: ${RCLONE_S3_CHUNK_SIZE}
+