add 10 gb 10 file batch

.gitattributes

 assays/a1/dataset/1gb.txt filter=lfs diff=lfs merge=lfs -text
 assays/a1/dataset/5gb.txt filter=lfs diff=lfs merge=lfs -text
 assays/a1/dataset/10gb.txt filter=lfs diff=lfs merge=lfs -text
+runs/10gbBatch/file10.bin filter=lfs diff=lfs merge=lfs -text
+runs/10gbBatch/file3.bin filter=lfs diff=lfs merge=lfs -text
+runs/10gbBatch/file5.bin filter=lfs diff=lfs merge=lfs -text
+runs/10gbBatch/file7.bin filter=lfs diff=lfs merge=lfs -text
+runs/10gbBatch/file8.bin filter=lfs diff=lfs merge=lfs -text
+runs/10gbBatch/file9.bin filter=lfs diff=lfs merge=lfs -text
+runs/10gbBatch/file1.bin filter=lfs diff=lfs merge=lfs -text
+runs/10gbBatch/file2.bin filter=lfs diff=lfs merge=lfs -text
+runs/10gbBatch/file4.bin filter=lfs diff=lfs merge=lfs -text
+runs/10gbBatch/file6.bin filter=lfs diff=lfs merge=lfs -text

workflows/createFiles.Fsx

+// open System
+// open System.IO
+
+// let createFile (fileName: string) (size: int64) =
+//     let bufferSize = 1024 * 1024 // 1MB buffer size
+//     let buffer = Array.zeroCreate<byte> bufferSize
+//     use fileStream = new FileStream(fileName, FileMode.Create, FileAccess.Write, FileShare.None)
+//     let rec writeData remainingSize =
+//         if remainingSize > 0L then
+//             let toWrite = int64 bufferSize |> min remainingSize
+//             fileStream.Write(buffer, 0, int toWrite)
+//             writeData (remainingSize - toWrite)
+//     writeData size
+//     printfn "Created file: %s" fileName
+
+// let currentDir = Directory.GetCurrentDirectory()
+// let outputDir = Path.Combine(currentDir, "runs", "10gbBatch")
+
+// // Ensure the output directory exists
+// Directory.CreateDirectory(outputDir) |> ignore
+
+// let fileSize = 10L * 1024L * 1024L * 1024L // 10 GB in bytes
+
+// [1 .. 10]
+// |> List.iter (fun i ->
+//     let fileName = Path.Combine(outputDir, sprintf "file%d.bin" i)
+//     createFile fileName fileSize
+//  )
+
+open System
+open System.IO
+open System.Threading.Tasks
+
+let createFile (fileName: string) (size: int64) =
+    let bufferSize = 1024 * 10024 // 1MB buffer size
+    let buffer = Array.zeroCreate<byte> bufferSize
+    use fileStream = new FileStream(fileName, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize, FileOptions.SequentialScan)
+    use bufferedStream = new BufferedStream(fileStream, bufferSize)
+    let rec writeData remainingSize =
+        if remainingSize > 0L then
+            let toWrite = int64 bufferSize |> min remainingSize
+            bufferedStream.Write(buffer, 0, int toWrite)
+            writeData (remainingSize - toWrite)
+    writeData size
+    printfn "Created file: %s" fileName
+
+let currentDir = Directory.GetCurrentDirectory()
+let outputDir = Path.Combine(currentDir, "runs", "10gbBatch")
+
+// Ensure the output directory exists
+Directory.CreateDirectory(outputDir) |> ignore
+
+let fileSize = 10L * 1024L * 1024L * 1024L // 10 GB in bytes
+
+let tasks =
+    [1 .. 10]
+    |> List.map (fun i ->
+        Task.Run(fun () ->
+            let fileName = Path.Combine(outputDir, sprintf "file%d.bin" i)
+            createFile fileName fileSize
+        ))
+
+Task.WhenAll(tasks) |> Async.AwaitTask |> Async.RunSynchronously |> ignore