WIP: Implement upload/download of trace records

src/ct/launch/launch.nim

@@ -2,6 +2,7 @@ import
   std/[strutils, strformat, osproc],
   ../../common/[ paths, types, intel_fix, install_utils, trace_index, start_utils ],
   ../utilities/[ git, env ],
+  ../online_sharing/trace_manager,
   ../cli/[ logging, list, help ],
   ../trace/[ replay, record, run, metadata ],
   ../codetracerconf,
@@ -101,15 +102,15 @@ proc runInitial*(conf: CodetracerConf) =
       notSupportedCommand($conf.cmd)
     of StartupCommand.upload:
       # similar to replay/console
-      notSupportedCommand($conf.cmd)
-      # eventually enable?
-      # uploadCommand(
-      #   conf.uploadLastTraceMatchingPattern,
-      #   conf.uploadTraceId,
-      #   conf.uploadTraceFolder,
-      #   replayInteractive)
+      uploadCommand(
+        conf.uploadLastTraceMatchingPattern,
+        conf.uploadTraceId,
+        conf.uploadTraceFolder,
+        replayInteractive)
     of StartupCommand.download:
-      notSupportedCommand($conf.cmd)
+      downloadCommand(conf.traceRegistryId)
+    of StartupCommand.cmdDelete:
+      deleteTraceCommand(conf.traceId, conf.controlId)
       # eventually enable?
       # downloadCommand(conf.traceRegistryId)
     # of StartupCommand.build:

src/ct/online_sharing/security_upload.nim

@@ -0,0 +1,68 @@
+import nimcrypto, zip/zipfiles, std/[ sequtils, strutils, strformat, os, osproc ]
+import ../../common/[ config, trace_index, lang ]
+
+proc generateSecurePassword*(): string =
+  var key: array[32, byte]
+  discard randomBytes(key)
+
+  result = key.mapIt(it.toHex(2)).join("")
+  return result
+
+proc pkcs7Pad*(data: seq[byte], blockSize: int): seq[byte] =
+  let padLen = blockSize - (data.len mod blockSize)
+  result = data & repeat(cast[byte](padLen), padLen)
+
+proc pkcs7Unpad*(data: seq[byte]): seq[byte] =
+  if data.len == 0:
+    raise newException(ValueError, "Data is empty, cannot unpad")
+
+  let padLen = int64(data[^1])  # Convert last byte to int64 safely
+  if padLen <= 0 or padLen > data.len:
+    raise newException(ValueError, "Invalid padding")
+
+  result = data[0 ..< data.len - padLen]
+
+func toBytes*(s: string): seq[byte] =
+  ## Convert a string to the corresponding byte sequence - since strings in
+  ## nim essentially are byte sequences without any particular encoding, this
+  ## simply copies the bytes without a null terminator
+  when nimvm:
+    var r = newSeq[byte](s.len)
+    for i, c in s:
+      r[i] = cast[byte](c)
+    r
+  else:
+    @(s.toOpenArrayByte(0, s.high))
+
+proc encryptZip(zipFile, password: string) =
+  var iv: seq[byte] = password.toBytes()[0..15]
+
+  var aes: CBC[aes256]
+  aes.init(password.toOpenArrayByte(0, len(password) - 1), iv)
+
+  var zipData = readFile(zipFile).toBytes()
+  var paddedData = pkcs7Pad(zipData, 16)
+  var encrypted = newSeq[byte](paddedData.len)
+
+  aes.encrypt(paddedData, encrypted.toOpenArray(0, len(encrypted) - 1))
+  writeFile(zipFile & ".enc", encrypted)
+
+proc zipFileWithEncryption*(inputFile: string, outputZip: string, password: string) =
+  var zip: ZipArchive
+  if not zip.open(outputZip, fmWrite):
+    raise newException(IOError, "Failed to create zip file: " & outputZip)
+
+  for file in walkDirRec(inputFile):
+    let relPath = file.relativePath(inputFile)
+    zip.addFile(relPath, file)
+
+  zip.close()
+  encryptZip(outputZip, password)
+  removeFile(outputZip)
+
+proc uploadEncyptedZip*(file: string): (string, int) =
+  # TODO: Plug in http client instead of curl
+  let config = loadConfig(folder=getCurrentDir(), inTest=false)
+  let cmd = &"curl -s -X POST -F \"file=@{file}.enc\" {config.webApiRoot}/upload"
+  let (output, exitCode) = execCmdEx(cmd)
+  (output, exitCode)

src/ct/online_sharing/trace_manager.nim

@@ -0,0 +1,100 @@
+import std/[ options, strutils, os, osproc, strformat, json ], ../trace/replay, ../codetracerconf, zip/zipfiles, nimcrypto
+import ../../common/[ config, trace_index, lang ]
+import ../utilities/language_detection
+import ../trace/[ storage_and_import, record ]
+import security_upload
+
+proc uploadCommand*(
+  patternArg: Option[string],
+  traceIdArg: Option[int],
+  traceFolderArg: Option[string],
+  interactive: bool
+) =
+  discard internalReplayOrUpload(patternArg, traceIdArg, traceFolderArg, interactive, command=StartupCommand.upload)
+
+
+proc decryptZip(encryptedFile: string, password: string, outputFile: string) =
+  var encData = readFile(encryptedFile).toBytes()
+  if encData.len < 16:
+    raise newException(ValueError, "Invalid encrypted data (too short)")
+
+  let iv = password.toBytes()[0 ..< 16]
+  let ciphertext = encData[16 .. ^1]
+  let key = password.toBytes()
+
+  var aes: CBC[aes256]
+  aes.init(key, iv)
+
+  var decrypted = newSeq[byte](encData.len)
+  aes.decrypt(encData, decrypted.toOpenArray(0, len(decrypted) - 1))
+
+  var depaddedData = pkcs7Unpad(decrypted)
+  writeFile(outputFile, depaddedData)
+
+proc unzipDecryptedFile(zipFile: string, outputDir: string): (string, int) =
+  var zip: ZipArchive
+  if not zip.open(zipFile, fmRead):
+    raise newException(IOError, "Failed to open decrypted ZIP: " & zipFile)
+
+  let traceId = trace_index.newID(false)
+  let outPath = outputDir / "trace-" & $traceId
+
+  createDir(outPath)
+  zip.extractAll(outPath)
+
+  zip.close()
+  return (outPath, traceId)
+
+proc downloadCommand*(traceRegistryId: string) =
+  # We expect a traceRegistryId to have <downloadId>::<passwordKey>
+  let stringSplit = traceRegistryId.split("//")
+  if stringSplit.len() != 3:
+    quit(1)
+  else:
+    let downloadId = stringSplit[1]
+    let password = stringSplit[2]
+    let zipPath = "/tmp/tmp.zip"
+    let config = loadConfig(folder=getCurrentDir(), inTest=false)
+    let localPath = "/tmp" / "tmp.zip.enc"
+    # TODO: Plug in an http client
+    let cmd = &"curl -s -o {localPath} {config.webApiRoot}/download?DownloadId={downloadId}"
+    let (output, exitCode) = execCmdEx(cmd)
+
+    decryptZip(localPath, password, zipPath)
+
+    let (traceFolder, traceId) = unzipDecryptedFile(zipPath, os.getHomeDir() / ".local" / "share" / "codetracer")
+    let tracePath = traceFolder / "trace.json"
+    let traceJson = parseJson(readFile(tracePath))
+    let traceMetadataPath = traceFolder / "trace_metadata.json"
+
+    var pathValue = ""
+
+    for item in traceJson:
+      if item.hasKey("Path"):
+        pathValue = item["Path"].getStr("")
+        break
+
+    let lang = detectLang(pathValue, LangUnknown)
+    discard importDbTrace(traceMetadataPath, traceId, lang, DB_SELF_CONTAINED_DEFAULT, traceRegistryId)
+
+    removeFile(localPath)
+    removeFile(zipPath)
+
+    echo traceId
+
+    quit(exitCode)
+
+proc deleteAndResetFields(id: int, test: bool) =
+  updateField(id, "remoteShareDownloadId", "", test)
+  updateField(id, "remoteShareControlId", "", test)
+  updateField(id, "remoteShareExpireTime", -1, test)
+
+proc deleteTraceCommand*(id: int, controlId: string) =
+  let config = loadConfig(folder=getCurrentDir(), inTest=false)
+  let cmd = &"curl -s {config.webApiRoot}/delete?ControlId={controlId}"
+  let (output, exitCode) = execCmdEx(cmd)
+
+  if exitCode == 0:
+    deleteAndResetFields(id, false)
+
+  quit(exitCode)

src/ct/trace/record.nim

@@ -54,7 +54,7 @@ proc recordSymbols(sourceDir: string, outputFolder: string, lang: Lang) =
 # it's still good to have an option/opt-out, so we leave that
 # as a flag in the internals, but not exposed to user yet
 # that's why for now it's hardcoded for db
-const DB_SELF_CONTAINED_DEFAULT = true
+const DB_SELF_CONTAINED_DEFAULT* = true
 
 # rr patches for ruby/other vm-s: not supported now, instead
 # in db backend support only direct traces

src/ct/trace/replay.nim

@@ -6,7 +6,7 @@ import std/[options ],
   shell,
   run
 
-proc internalReplayOrUpload(
+proc internalReplayOrUpload*(
   patternArg: Option[string],
   traceIdArg: Option[int],
   traceFolderArg: Option[string],

src/ct/trace/storage_and_import.nim

@@ -2,6 +2,7 @@ import
   std/[os, json, strutils, strformat, sets, algorithm],
   ../../common/[trace_index, lang, types, paths],
   ../utilities/git,
+  ../online_sharing/security_upload,
   json_serialization
 
 proc storeTraceFiles(paths: seq[string], traceFolder: string, lang: Lang) =
@@ -78,10 +79,12 @@ proc processSourceFoldersList*(folderSet: HashSet[string], programDir: string =
 
 
 proc importDbTrace*(
-    traceMetadataPath: string,
-    traceIdArg: int,
-    lang: Lang = LangNoir,
-    selfContained: bool = true): Trace =
+  traceMetadataPath: string,
+  traceIdArg: int,
+  lang: Lang = LangNoir,
+  selfContained: bool = true,
+  downloadKey: string = ""
+): Trace =
   let rawTraceMetadata = readFile(traceMetadataPath)
   let untypedJson = parseJson(rawTraceMetadata)
   let program = untypedJson{"program"}.getStr()
@@ -155,8 +158,32 @@ proc importDbTrace*(
     calltrace = true,
     # for now always use FullRecord for db-backend
     # and ignore possible env var override
-    calltraceMode = CalltraceMode.FullRecord)
+    calltraceMode = CalltraceMode.FullRecord,
+    downloadKey = downloadKey)
 
 proc uploadTrace*(trace: Trace) =
-  echo "error: uploading traces not supported currently!"
-  quit(1)
+  let outputZip = trace.outputFolder / "tmp.zip"
+  let aesKey = generateSecurePassword()
+
+  zipFileWithEncryption(trace.outputFolder, outputZip, aesKey)
+
+  let (output, exitCode) = uploadEncyptedZip(outputZip)
+  let jsonMessage = parseJson(output)
+  let downloadKey = trace.program & "//" & jsonMessage["DownloadId"].getStr("") & "//" & aesKey
+
+  if jsonMessage["DownloadId"].getStr("") notin @["", "Errored"]:
+
+    updateField(trace.id, "remoteShareDownloadId", downloadKey, false)
+    updateField(trace.id, "remoteShareControlId", jsonMessage["ControlId"].getStr(""), false)
+    updateField(trace.id, "remoteShareExpireTime", jsonMessage["Expires"].getInt(), false)
+
+    echo downloadKey
+    echo jsonMessage["ControlId"].getStr("")
+    echo jsonMessage["Expires"].getInt()
+
+  else:
+    echo downloadKey
+
+  removeFile(outputZip & ".enc")
+
+  quit(exitCode)