diff --git a/.gitattributes b/.gitattributes index 297fe601513029b6781e13275e087946957718a8..9b93f32d8ce3d801734fce2efb21628cdbe627d3 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,13 @@ assays/a1/dataset/1gb.txt filter=lfs diff=lfs merge=lfs -text assays/a1/dataset/5gb.txt filter=lfs diff=lfs merge=lfs -text assays/a1/dataset/10gb.txt filter=lfs diff=lfs merge=lfs -text +runs/10gbBatch/file10.bin filter=lfs diff=lfs merge=lfs -text +runs/10gbBatch/file3.bin filter=lfs diff=lfs merge=lfs -text +runs/10gbBatch/file5.bin filter=lfs diff=lfs merge=lfs -text +runs/10gbBatch/file7.bin filter=lfs diff=lfs merge=lfs -text +runs/10gbBatch/file8.bin filter=lfs diff=lfs merge=lfs -text +runs/10gbBatch/file9.bin filter=lfs diff=lfs merge=lfs -text +runs/10gbBatch/file1.bin filter=lfs diff=lfs merge=lfs -text +runs/10gbBatch/file2.bin filter=lfs diff=lfs merge=lfs -text +runs/10gbBatch/file4.bin filter=lfs diff=lfs merge=lfs -text +runs/10gbBatch/file6.bin filter=lfs diff=lfs merge=lfs -text diff --git a/runs/10gbBatch/file1.bin b/runs/10gbBatch/file1.bin new file mode 100644 index 0000000000000000000000000000000000000000..73e777b4a480b4d9ddd7213e8332b7cd6771afc2 --- /dev/null +++ b/runs/10gbBatch/file1.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:732377e7f4a2abdc13ddfa1eb4c9c497fd2a2b294674d056cf51581b47dd586d +size 10737418240 diff --git a/runs/10gbBatch/file10.bin b/runs/10gbBatch/file10.bin new file mode 100644 index 0000000000000000000000000000000000000000..73e777b4a480b4d9ddd7213e8332b7cd6771afc2 --- /dev/null +++ b/runs/10gbBatch/file10.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:732377e7f4a2abdc13ddfa1eb4c9c497fd2a2b294674d056cf51581b47dd586d +size 10737418240 diff --git a/runs/10gbBatch/file2.bin b/runs/10gbBatch/file2.bin new file mode 100644 index 0000000000000000000000000000000000000000..73e777b4a480b4d9ddd7213e8332b7cd6771afc2 --- /dev/null +++ b/runs/10gbBatch/file2.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:732377e7f4a2abdc13ddfa1eb4c9c497fd2a2b294674d056cf51581b47dd586d +size 10737418240 diff --git a/runs/10gbBatch/file3.bin b/runs/10gbBatch/file3.bin new file mode 100644 index 0000000000000000000000000000000000000000..73e777b4a480b4d9ddd7213e8332b7cd6771afc2 --- /dev/null +++ b/runs/10gbBatch/file3.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:732377e7f4a2abdc13ddfa1eb4c9c497fd2a2b294674d056cf51581b47dd586d +size 10737418240 diff --git a/runs/10gbBatch/file4.bin b/runs/10gbBatch/file4.bin new file mode 100644 index 0000000000000000000000000000000000000000..73e777b4a480b4d9ddd7213e8332b7cd6771afc2 --- /dev/null +++ b/runs/10gbBatch/file4.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:732377e7f4a2abdc13ddfa1eb4c9c497fd2a2b294674d056cf51581b47dd586d +size 10737418240 diff --git a/runs/10gbBatch/file5.bin b/runs/10gbBatch/file5.bin new file mode 100644 index 0000000000000000000000000000000000000000..73e777b4a480b4d9ddd7213e8332b7cd6771afc2 --- /dev/null +++ b/runs/10gbBatch/file5.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:732377e7f4a2abdc13ddfa1eb4c9c497fd2a2b294674d056cf51581b47dd586d +size 10737418240 diff --git a/runs/10gbBatch/file6.bin b/runs/10gbBatch/file6.bin new file mode 100644 index 0000000000000000000000000000000000000000..73e777b4a480b4d9ddd7213e8332b7cd6771afc2 --- /dev/null +++ b/runs/10gbBatch/file6.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:732377e7f4a2abdc13ddfa1eb4c9c497fd2a2b294674d056cf51581b47dd586d +size 10737418240 diff --git a/runs/10gbBatch/file7.bin b/runs/10gbBatch/file7.bin new file mode 100644 index 0000000000000000000000000000000000000000..73e777b4a480b4d9ddd7213e8332b7cd6771afc2 --- /dev/null +++ b/runs/10gbBatch/file7.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:732377e7f4a2abdc13ddfa1eb4c9c497fd2a2b294674d056cf51581b47dd586d +size 10737418240 diff --git a/runs/10gbBatch/file8.bin b/runs/10gbBatch/file8.bin new file mode 100644 index 0000000000000000000000000000000000000000..73e777b4a480b4d9ddd7213e8332b7cd6771afc2 --- /dev/null +++ b/runs/10gbBatch/file8.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:732377e7f4a2abdc13ddfa1eb4c9c497fd2a2b294674d056cf51581b47dd586d +size 10737418240 diff --git a/runs/10gbBatch/file9.bin b/runs/10gbBatch/file9.bin new file mode 100644 index 0000000000000000000000000000000000000000..73e777b4a480b4d9ddd7213e8332b7cd6771afc2 --- /dev/null +++ b/runs/10gbBatch/file9.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:732377e7f4a2abdc13ddfa1eb4c9c497fd2a2b294674d056cf51581b47dd586d +size 10737418240 diff --git a/workflows/createFiles.Fsx b/workflows/createFiles.Fsx new file mode 100644 index 0000000000000000000000000000000000000000..c94eb3226c1dde49f1b6bdd7755cfc5d2b11cb4e --- /dev/null +++ b/workflows/createFiles.Fsx @@ -0,0 +1,63 @@ +// open System +// open System.IO + +// let createFile (fileName: string) (size: int64) = +// let bufferSize = 1024 * 1024 // 1MB buffer size +// let buffer = Array.zeroCreate<byte> bufferSize +// use fileStream = new FileStream(fileName, FileMode.Create, FileAccess.Write, FileShare.None) +// let rec writeData remainingSize = +// if remainingSize > 0L then +// let toWrite = int64 bufferSize |> min remainingSize +// fileStream.Write(buffer, 0, int toWrite) +// writeData (remainingSize - toWrite) +// writeData size +// printfn "Created file: %s" fileName + +// let currentDir = Directory.GetCurrentDirectory() +// let outputDir = Path.Combine(currentDir, "runs", "10gbBatch") + +// // Ensure the output directory exists +// Directory.CreateDirectory(outputDir) |> ignore + +// let fileSize = 10L * 1024L * 1024L * 1024L // 10 GB in bytes + +// [1 .. 10] +// |> List.iter (fun i -> +// let fileName = Path.Combine(outputDir, sprintf "file%d.bin" i) +// createFile fileName fileSize +// ) + +open System +open System.IO +open System.Threading.Tasks + +let createFile (fileName: string) (size: int64) = + let bufferSize = 1024 * 10024 // 1MB buffer size + let buffer = Array.zeroCreate<byte> bufferSize + use fileStream = new FileStream(fileName, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize, FileOptions.SequentialScan) + use bufferedStream = new BufferedStream(fileStream, bufferSize) + let rec writeData remainingSize = + if remainingSize > 0L then + let toWrite = int64 bufferSize |> min remainingSize + bufferedStream.Write(buffer, 0, int toWrite) + writeData (remainingSize - toWrite) + writeData size + printfn "Created file: %s" fileName + +let currentDir = Directory.GetCurrentDirectory() +let outputDir = Path.Combine(currentDir, "runs", "10gbBatch") + +// Ensure the output directory exists +Directory.CreateDirectory(outputDir) |> ignore + +let fileSize = 10L * 1024L * 1024L * 1024L // 10 GB in bytes + +let tasks = + [1 .. 10] + |> List.map (fun i -> + Task.Run(fun () -> + let fileName = Path.Combine(outputDir, sprintf "file%d.bin" i) + createFile fileName fileSize + )) + +Task.WhenAll(tasks) |> Async.AwaitTask |> Async.RunSynchronously |> ignore