From 1e926b360394034bd0a208f88a7abb031d0ab846 Mon Sep 17 00:00:00 2001 From: David <zimmer@csbiology.de> Date: Sun, 30 Jun 2024 10:15:11 +0200 Subject: [PATCH] add 10 gb 10 file batch --- .gitattributes | 10 +++++++ runs/10gbBatch/file1.bin | 3 ++ runs/10gbBatch/file10.bin | 3 ++ runs/10gbBatch/file2.bin | 3 ++ runs/10gbBatch/file3.bin | 3 ++ runs/10gbBatch/file4.bin | 3 ++ runs/10gbBatch/file5.bin | 3 ++ runs/10gbBatch/file6.bin | 3 ++ runs/10gbBatch/file7.bin | 3 ++ runs/10gbBatch/file8.bin | 3 ++ runs/10gbBatch/file9.bin | 3 ++ workflows/createFiles.Fsx | 63 +++++++++++++++++++++++++++++++++++++++ 12 files changed, 103 insertions(+) create mode 100644 runs/10gbBatch/file1.bin create mode 100644 runs/10gbBatch/file10.bin create mode 100644 runs/10gbBatch/file2.bin create mode 100644 runs/10gbBatch/file3.bin create mode 100644 runs/10gbBatch/file4.bin create mode 100644 runs/10gbBatch/file5.bin create mode 100644 runs/10gbBatch/file6.bin create mode 100644 runs/10gbBatch/file7.bin create mode 100644 runs/10gbBatch/file8.bin create mode 100644 runs/10gbBatch/file9.bin create mode 100644 workflows/createFiles.Fsx diff --git a/.gitattributes b/.gitattributes index 297fe60..9b93f32 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,13 @@ assays/a1/dataset/1gb.txt filter=lfs diff=lfs merge=lfs -text assays/a1/dataset/5gb.txt filter=lfs diff=lfs merge=lfs -text assays/a1/dataset/10gb.txt filter=lfs diff=lfs merge=lfs -text +runs/10gbBatch/file10.bin filter=lfs diff=lfs merge=lfs -text +runs/10gbBatch/file3.bin filter=lfs diff=lfs merge=lfs -text +runs/10gbBatch/file5.bin filter=lfs diff=lfs merge=lfs -text +runs/10gbBatch/file7.bin filter=lfs diff=lfs merge=lfs -text +runs/10gbBatch/file8.bin filter=lfs diff=lfs merge=lfs -text +runs/10gbBatch/file9.bin filter=lfs diff=lfs merge=lfs -text +runs/10gbBatch/file1.bin filter=lfs diff=lfs merge=lfs -text +runs/10gbBatch/file2.bin filter=lfs diff=lfs merge=lfs -text +runs/10gbBatch/file4.bin filter=lfs diff=lfs merge=lfs -text +runs/10gbBatch/file6.bin filter=lfs diff=lfs merge=lfs -text diff --git a/runs/10gbBatch/file1.bin b/runs/10gbBatch/file1.bin new file mode 100644 index 0000000..73e777b --- /dev/null +++ b/runs/10gbBatch/file1.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:732377e7f4a2abdc13ddfa1eb4c9c497fd2a2b294674d056cf51581b47dd586d +size 10737418240 diff --git a/runs/10gbBatch/file10.bin b/runs/10gbBatch/file10.bin new file mode 100644 index 0000000..73e777b --- /dev/null +++ b/runs/10gbBatch/file10.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:732377e7f4a2abdc13ddfa1eb4c9c497fd2a2b294674d056cf51581b47dd586d +size 10737418240 diff --git a/runs/10gbBatch/file2.bin b/runs/10gbBatch/file2.bin new file mode 100644 index 0000000..73e777b --- /dev/null +++ b/runs/10gbBatch/file2.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:732377e7f4a2abdc13ddfa1eb4c9c497fd2a2b294674d056cf51581b47dd586d +size 10737418240 diff --git a/runs/10gbBatch/file3.bin b/runs/10gbBatch/file3.bin new file mode 100644 index 0000000..73e777b --- /dev/null +++ b/runs/10gbBatch/file3.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:732377e7f4a2abdc13ddfa1eb4c9c497fd2a2b294674d056cf51581b47dd586d +size 10737418240 diff --git a/runs/10gbBatch/file4.bin b/runs/10gbBatch/file4.bin new file mode 100644 index 0000000..73e777b --- /dev/null +++ b/runs/10gbBatch/file4.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:732377e7f4a2abdc13ddfa1eb4c9c497fd2a2b294674d056cf51581b47dd586d +size 10737418240 diff --git a/runs/10gbBatch/file5.bin b/runs/10gbBatch/file5.bin new file mode 100644 index 0000000..73e777b --- /dev/null +++ b/runs/10gbBatch/file5.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:732377e7f4a2abdc13ddfa1eb4c9c497fd2a2b294674d056cf51581b47dd586d +size 10737418240 diff --git a/runs/10gbBatch/file6.bin b/runs/10gbBatch/file6.bin new file mode 100644 index 0000000..73e777b --- /dev/null +++ b/runs/10gbBatch/file6.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:732377e7f4a2abdc13ddfa1eb4c9c497fd2a2b294674d056cf51581b47dd586d +size 10737418240 diff --git a/runs/10gbBatch/file7.bin b/runs/10gbBatch/file7.bin new file mode 100644 index 0000000..73e777b --- /dev/null +++ b/runs/10gbBatch/file7.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:732377e7f4a2abdc13ddfa1eb4c9c497fd2a2b294674d056cf51581b47dd586d +size 10737418240 diff --git a/runs/10gbBatch/file8.bin b/runs/10gbBatch/file8.bin new file mode 100644 index 0000000..73e777b --- /dev/null +++ b/runs/10gbBatch/file8.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:732377e7f4a2abdc13ddfa1eb4c9c497fd2a2b294674d056cf51581b47dd586d +size 10737418240 diff --git a/runs/10gbBatch/file9.bin b/runs/10gbBatch/file9.bin new file mode 100644 index 0000000..73e777b --- /dev/null +++ b/runs/10gbBatch/file9.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:732377e7f4a2abdc13ddfa1eb4c9c497fd2a2b294674d056cf51581b47dd586d +size 10737418240 diff --git a/workflows/createFiles.Fsx b/workflows/createFiles.Fsx new file mode 100644 index 0000000..c94eb32 --- /dev/null +++ b/workflows/createFiles.Fsx @@ -0,0 +1,63 @@ +// open System +// open System.IO + +// let createFile (fileName: string) (size: int64) = +// let bufferSize = 1024 * 1024 // 1MB buffer size +// let buffer = Array.zeroCreate<byte> bufferSize +// use fileStream = new FileStream(fileName, FileMode.Create, FileAccess.Write, FileShare.None) +// let rec writeData remainingSize = +// if remainingSize > 0L then +// let toWrite = int64 bufferSize |> min remainingSize +// fileStream.Write(buffer, 0, int toWrite) +// writeData (remainingSize - toWrite) +// writeData size +// printfn "Created file: %s" fileName + +// let currentDir = Directory.GetCurrentDirectory() +// let outputDir = Path.Combine(currentDir, "runs", "10gbBatch") + +// // Ensure the output directory exists +// Directory.CreateDirectory(outputDir) |> ignore + +// let fileSize = 10L * 1024L * 1024L * 1024L // 10 GB in bytes + +// [1 .. 10] +// |> List.iter (fun i -> +// let fileName = Path.Combine(outputDir, sprintf "file%d.bin" i) +// createFile fileName fileSize +// ) + +open System +open System.IO +open System.Threading.Tasks + +let createFile (fileName: string) (size: int64) = + let bufferSize = 1024 * 10024 // 1MB buffer size + let buffer = Array.zeroCreate<byte> bufferSize + use fileStream = new FileStream(fileName, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize, FileOptions.SequentialScan) + use bufferedStream = new BufferedStream(fileStream, bufferSize) + let rec writeData remainingSize = + if remainingSize > 0L then + let toWrite = int64 bufferSize |> min remainingSize + bufferedStream.Write(buffer, 0, int toWrite) + writeData (remainingSize - toWrite) + writeData size + printfn "Created file: %s" fileName + +let currentDir = Directory.GetCurrentDirectory() +let outputDir = Path.Combine(currentDir, "runs", "10gbBatch") + +// Ensure the output directory exists +Directory.CreateDirectory(outputDir) |> ignore + +let fileSize = 10L * 1024L * 1024L * 1024L // 10 GB in bytes + +let tasks = + [1 .. 10] + |> List.map (fun i -> + Task.Run(fun () -> + let fileName = Path.Combine(outputDir, sprintf "file%d.bin" i) + createFile fileName fileSize + )) + +Task.WhenAll(tasks) |> Async.AwaitTask |> Async.RunSynchronously |> ignore -- GitLab