diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..e79f24b4ad453e7c4cb335dd8bc1e06a1f4a4e68 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +*.csv filter=lfs diff=lfs merge=lfs -text diff --git a/Acme/cluster_full_log.csv b/Acme/cluster_full_log.csv new file mode 100644 index 0000000000000000000000000000000000000000..10bbd4b0d0ce5157bf1dba98099470a266bb47c8 --- /dev/null +++ b/Acme/cluster_full_log.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41cb67764a8bce6e073fd369da4d6125710358b78a8601b6044ca3a457e52a6a +size 14410276 diff --git a/Acme/cluster_log.csv b/Acme/cluster_log.csv new file mode 100644 index 0000000000000000000000000000000000000000..10bbd4b0d0ce5157bf1dba98099470a266bb47c8 --- /dev/null +++ b/Acme/cluster_log.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41cb67764a8bce6e073fd369da4d6125710358b78a8601b6044ca3a457e52a6a +size 14410276 diff --git a/Acme/vc_config.csv b/Acme/vc_config.csv new file mode 100644 index 0000000000000000000000000000000000000000..5b636f997aee55a3d07203de63ee3c459a18cb63 --- /dev/null +++ b/Acme/vc_config.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb95968b1185b0f69f27abece683ce1081de988cd2b7001047a337de2ef97ffa +size 14 diff --git a/Debug/cluster_full_log.csv b/Debug/cluster_full_log.csv new file mode 100644 index 0000000000000000000000000000000000000000..8f5cae8909adb2a0cade7e50baae308875f31406 --- /dev/null +++ b/Debug/cluster_full_log.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f46b19f55b17ab8324eab3b712ffee5909c8f93273d219804ce7ea25e26d1988 +size 158656967 diff --git a/Debug/cluster_log.csv b/Debug/cluster_log.csv new file mode 100644 index 0000000000000000000000000000000000000000..6e8289ae9533e5127a68a1fe2bc6d6aad9724dba --- /dev/null +++ b/Debug/cluster_log.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6aeabe07589d153acc54f958bfb7c6582c1d93993f4dab1a76332d672c65405f +size 248210949 diff --git a/Debug/cluster_throughput.csv b/Debug/cluster_throughput.csv new file mode 100644 index 0000000000000000000000000000000000000000..ac45c4ccbdc80a23fdbc9becf82165c4547e3d72 --- /dev/null +++ b/Debug/cluster_throughput.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26b1cd6c93431910fcb84027e8a662fb9f1c588a27e0b2dcc8fdf2ab5a7120bf +size 1206927 diff --git a/Debug/vc_config.csv b/Debug/vc_config.csv new file mode 100644 index 0000000000000000000000000000000000000000..538816477cd0c0f30bdbaa19c3cf6db981c152bf --- /dev/null +++ b/Debug/vc_config.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72a16cafd27c064ecaef202da90a84eb382a637ca854e0a28c8247eb7470b3ba +size 176 diff --git a/Debug/vc_config.old.csv b/Debug/vc_config.old.csv new file mode 100644 index 0000000000000000000000000000000000000000..d8511f7e61ddd3ee912096dec0d086484a068b0a --- /dev/null +++ b/Debug/vc_config.old.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f385122d488bf088812246793a9b2e3d3379170f9905d99ac8b6baabd8cb5c7d +size 39 diff --git a/Debug/vc_config_best.csv b/Debug/vc_config_best.csv new file mode 100644 index 0000000000000000000000000000000000000000..d9e744a81d8734d12ded0e07a266643657334262 --- /dev/null +++ b/Debug/vc_config_best.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d0dc91ffab6ba27514932f380830bea9dfeb020a339ccfbff7c108b5a0c0d3f +size 31 diff --git a/Med(base)/cluster_full_log.csv b/Med(base)/cluster_full_log.csv new file mode 100644 index 0000000000000000000000000000000000000000..141219c7e063b1f3931899f54ead3481a9a71581 --- /dev/null +++ b/Med(base)/cluster_full_log.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82ee277b2961d8cbd77c609258ff04742b70585ae23760b9629231bf0ff718d1 +size 158641436 diff --git a/Med(base)/cluster_log.csv b/Med(base)/cluster_log.csv new file mode 100644 index 0000000000000000000000000000000000000000..6e8289ae9533e5127a68a1fe2bc6d6aad9724dba --- /dev/null +++ b/Med(base)/cluster_log.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6aeabe07589d153acc54f958bfb7c6582c1d93993f4dab1a76332d672c65405f +size 248210949 diff --git a/Med(base)/cluster_throughput.csv b/Med(base)/cluster_throughput.csv new file mode 100644 index 0000000000000000000000000000000000000000..ac45c4ccbdc80a23fdbc9becf82165c4547e3d72 --- /dev/null +++ b/Med(base)/cluster_throughput.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26b1cd6c93431910fcb84027e8a662fb9f1c588a27e0b2dcc8fdf2ab5a7120bf +size 1206927 diff --git a/Med(base)/vc_config.csv b/Med(base)/vc_config.csv new file mode 100644 index 0000000000000000000000000000000000000000..95b46fc5e3a1a4dcfcfdc3518c28179b76455ec8 --- /dev/null +++ b/Med(base)/vc_config.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e75af64b27ad88c0802bc9f51bf6f4adf33dd1308cf9b98510e46da9e73ca12 +size 105 diff --git a/Med(base)/vc_config.old.csv b/Med(base)/vc_config.old.csv new file mode 100644 index 0000000000000000000000000000000000000000..d8511f7e61ddd3ee912096dec0d086484a068b0a --- /dev/null +++ b/Med(base)/vc_config.old.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f385122d488bf088812246793a9b2e3d3379170f9905d99ac8b6baabd8cb5c7d +size 39 diff --git a/Med(base)/vc_config_best.csv b/Med(base)/vc_config_best.csv new file mode 100644 index 0000000000000000000000000000000000000000..d9e744a81d8734d12ded0e07a266643657334262 --- /dev/null +++ b/Med(base)/vc_config_best.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d0dc91ffab6ba27514932f380830bea9dfeb020a339ccfbff7c108b5a0c0d3f +size 31 diff --git a/Med/cluster_full_log.csv b/Med/cluster_full_log.csv new file mode 100644 index 0000000000000000000000000000000000000000..141219c7e063b1f3931899f54ead3481a9a71581 --- /dev/null +++ b/Med/cluster_full_log.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82ee277b2961d8cbd77c609258ff04742b70585ae23760b9629231bf0ff718d1 +size 158641436 diff --git a/Med/cluster_log.csv b/Med/cluster_log.csv new file mode 100644 index 0000000000000000000000000000000000000000..6e8289ae9533e5127a68a1fe2bc6d6aad9724dba --- /dev/null +++ b/Med/cluster_log.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6aeabe07589d153acc54f958bfb7c6582c1d93993f4dab1a76332d672c65405f +size 248210949 diff --git a/Med/cluster_throughput.csv b/Med/cluster_throughput.csv new file mode 100644 index 0000000000000000000000000000000000000000..ac45c4ccbdc80a23fdbc9becf82165c4547e3d72 --- /dev/null +++ b/Med/cluster_throughput.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26b1cd6c93431910fcb84027e8a662fb9f1c588a27e0b2dcc8fdf2ab5a7120bf +size 1206927 diff --git a/Med/vc_config.csv b/Med/vc_config.csv new file mode 100644 index 0000000000000000000000000000000000000000..175cfa4056111060d351eb4dd2f2d2d1c8862c71 --- /dev/null +++ b/Med/vc_config.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2af70b24bff19f6bb05f30fffb8f8099ea1e1fa2de54b3f23749d0594d0b0cc +size 122 diff --git a/Med/vc_config.old.csv b/Med/vc_config.old.csv new file mode 100644 index 0000000000000000000000000000000000000000..d8511f7e61ddd3ee912096dec0d086484a068b0a --- /dev/null +++ b/Med/vc_config.old.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f385122d488bf088812246793a9b2e3d3379170f9905d99ac8b6baabd8cb5c7d +size 39 diff --git a/Med/vc_config_best.csv b/Med/vc_config_best.csv new file mode 100644 index 0000000000000000000000000000000000000000..d9e744a81d8734d12ded0e07a266643657334262 --- /dev/null +++ b/Med/vc_config_best.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d0dc91ffab6ba27514932f380830bea9dfeb020a339ccfbff7c108b5a0c0d3f +size 31 diff --git a/Philly/cluster_full_log.csv b/Philly/cluster_full_log.csv new file mode 100644 index 0000000000000000000000000000000000000000..de00c8785423038859ac8cfb3d1d18e58b1408e2 --- /dev/null +++ b/Philly/cluster_full_log.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:183d7205e6273ae9178f10bab16a838c9563526283c0e951f254992b0475b8ad +size 19771533 diff --git a/Philly/cluster_log.csv b/Philly/cluster_log.csv new file mode 100644 index 0000000000000000000000000000000000000000..e2fe25be12dccdaaa35936af95e2e2e635dd6386 --- /dev/null +++ b/Philly/cluster_log.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f3f87b1611b2790db0b9b045f28fabc63796011dda4546d991baaedca8670cc +size 14003345 diff --git a/Philly/cluster_throughput.csv b/Philly/cluster_throughput.csv new file mode 100644 index 0000000000000000000000000000000000000000..b8f0d0d3caba9b5f55106e1626b57ab8f56110db --- /dev/null +++ b/Philly/cluster_throughput.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e13084518fa590a0d83ef409c4bc3fdd6da62c76c3340efd8f2e5e77e51a22f +size 242304 diff --git a/Philly/vc_config.csv b/Philly/vc_config.csv new file mode 100644 index 0000000000000000000000000000000000000000..c71ba4de1a93ba36e0436e289df9c489daa0ea64 --- /dev/null +++ b/Philly/vc_config.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e34469fdf86d7ac928ee7b003c13b9a65606fe854716af7b91e6fc79ac143fb +size 16 diff --git a/Pollux/cluster_full_log_1.csv b/Pollux/cluster_full_log_1.csv new file mode 100644 index 0000000000000000000000000000000000000000..4183c306f86ffb7b30aed0d4e750be17ad495c3d --- /dev/null +++ b/Pollux/cluster_full_log_1.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a3aba6bcd81d82560434b7e1faf18c192b91eb605803c649ee29ac6c5f33cdf +size 52791 diff --git a/Pollux/cluster_full_log_2.csv b/Pollux/cluster_full_log_2.csv new file mode 100644 index 0000000000000000000000000000000000000000..8f887160707520c53ddd14fdc00c130de54b3e03 --- /dev/null +++ b/Pollux/cluster_full_log_2.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d151b72b8f3937c9108392322d0fb397791217da8de7c7b9d3f4ce67c05b680b +size 53001 diff --git a/Pollux/cluster_full_log_3.csv b/Pollux/cluster_full_log_3.csv new file mode 100644 index 0000000000000000000000000000000000000000..7e6aeac917f2c6f0099bee48ce3a64bf3b684160 --- /dev/null +++ b/Pollux/cluster_full_log_3.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ec1d641f2f262068519cab709c9b42959191acdf218cceb6ae84a31b21c7850 +size 52921 diff --git a/Pollux/cluster_full_log_4.csv b/Pollux/cluster_full_log_4.csv new file mode 100644 index 0000000000000000000000000000000000000000..187ebd2a582b87a52e872fc632bd096bdb60aa04 --- /dev/null +++ b/Pollux/cluster_full_log_4.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be6edc8cb73393339fc0fe0e9e1fa9615e8bf7abb2630e5cddae6845af4c580f +size 52922 diff --git a/Pollux/cluster_full_log_5.csv b/Pollux/cluster_full_log_5.csv new file mode 100644 index 0000000000000000000000000000000000000000..b17fdee82dd94d36a9e5e74da2124f8b73169346 --- /dev/null +++ b/Pollux/cluster_full_log_5.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96917a3e989080c8731ef0c28b0421e54cf89cd6c545602f8dadfa6380318052 +size 52977 diff --git a/Pollux/cluster_full_log_6.csv b/Pollux/cluster_full_log_6.csv new file mode 100644 index 0000000000000000000000000000000000000000..bdc0b88a26fdc93a66f15c2deb14fe216e82b143 --- /dev/null +++ b/Pollux/cluster_full_log_6.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c376c76baea04ab2f31df0ee31e5b8bf695c0e72a8649897f61e1470ed527ba9 +size 52955 diff --git a/Pollux/cluster_full_log_7.csv b/Pollux/cluster_full_log_7.csv new file mode 100644 index 0000000000000000000000000000000000000000..b439405eb448fde724da70da26914cc69d266d0b --- /dev/null +++ b/Pollux/cluster_full_log_7.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60cebd0ea413d8e10cf074fb617d87db58b4f64780a3b7314d57e28e6f7c010c +size 52925 diff --git a/Pollux/cluster_full_log_8.csv b/Pollux/cluster_full_log_8.csv new file mode 100644 index 0000000000000000000000000000000000000000..fdf7dc01d80776925495fb4d062cb7d1a83a84ee --- /dev/null +++ b/Pollux/cluster_full_log_8.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ec259c02cab96e3d6df775a8c15aec663c107877e20eae41b151b22b9c01b23 +size 53035 diff --git a/Pollux/pollux_jobs.csv b/Pollux/pollux_jobs.csv new file mode 100644 index 0000000000000000000000000000000000000000..4eefbeb93af0ff9a80f629f80bfe47e4b8c93fc5 --- /dev/null +++ b/Pollux/pollux_jobs.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:592a1bc10bc235b85f8a919f8e86566f606b456f97bd3c872e1f1b6e369b61b0 +size 461 diff --git a/Pollux/vc_config.csv b/Pollux/vc_config.csv new file mode 100644 index 0000000000000000000000000000000000000000..b6b75662c65a36e1961118a3e9d0c73ab15945d5 --- /dev/null +++ b/Pollux/vc_config.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:277a6c0950325ac4ff80d72857393a7f58eadfe16e62207a9c0494ce1d1edc6e +size 16 diff --git a/Real/cluster_full_log.csv b/Real/cluster_full_log.csv new file mode 100644 index 0000000000000000000000000000000000000000..1da2c267bcc0c93ac86dff4f6a7dbafa6207d18b --- /dev/null +++ b/Real/cluster_full_log.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c34a3a33590d95bf910da0c845719248b411127c8050b3c0c7ef31a78186fca +size 9759 diff --git a/Real/vc_config.csv b/Real/vc_config.csv new file mode 100644 index 0000000000000000000000000000000000000000..021954f9f3e4fd9abfdbb83a6a1610ebed8215bf --- /dev/null +++ b/Real/vc_config.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c07379d4da9deb2db7cb29e8f515ca611fc4481b190e5c5f631b90656ebd7715 +size 14 diff --git a/RealCont/cluster_full_log.csv b/RealCont/cluster_full_log.csv new file mode 100644 index 0000000000000000000000000000000000000000..9952a80bb89dc447c9b325573eaaeb68b9689b74 --- /dev/null +++ b/RealCont/cluster_full_log.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43b77588536d6b51e12f94715fb802b66db945b20c616ca114275e490fed375c +size 11952 diff --git a/RealCont/vc_config.csv b/RealCont/vc_config.csv new file mode 100644 index 0000000000000000000000000000000000000000..021954f9f3e4fd9abfdbb83a6a1610ebed8215bf --- /dev/null +++ b/RealCont/vc_config.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c07379d4da9deb2db7cb29e8f515ca611fc4481b190e5c5f631b90656ebd7715 +size 14 diff --git a/Saturn/cluster_full_log.csv b/Saturn/cluster_full_log.csv new file mode 100644 index 0000000000000000000000000000000000000000..ee790efc70b0c21fa918828b3f2e93dda87a4bd7 --- /dev/null +++ b/Saturn/cluster_full_log.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdfddf313e2783f8bffb1ddee94c0aea954a981e9f3cb6f01519a35369444044 +size 133500962 diff --git a/Saturn/cluster_log.csv b/Saturn/cluster_log.csv new file mode 100644 index 0000000000000000000000000000000000000000..6e8289ae9533e5127a68a1fe2bc6d6aad9724dba --- /dev/null +++ b/Saturn/cluster_log.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6aeabe07589d153acc54f958bfb7c6582c1d93993f4dab1a76332d672c65405f +size 248210949 diff --git a/Saturn/cluster_throughput.csv b/Saturn/cluster_throughput.csv new file mode 100644 index 0000000000000000000000000000000000000000..ac45c4ccbdc80a23fdbc9becf82165c4547e3d72 --- /dev/null +++ b/Saturn/cluster_throughput.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26b1cd6c93431910fcb84027e8a662fb9f1c588a27e0b2dcc8fdf2ab5a7120bf +size 1206927 diff --git a/Saturn/vc_config.csv b/Saturn/vc_config.csv new file mode 100644 index 0000000000000000000000000000000000000000..a6f09e87dbe00a510b1ac589ca1f5541bcf11f47 --- /dev/null +++ b/Saturn/vc_config.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cac3755947a25d618e49bb1078c68afbca4f4d74c0e8780abdc359baf2b61c70 +size 176 diff --git a/Venus/cluster_full_log.csv b/Venus/cluster_full_log.csv new file mode 100644 index 0000000000000000000000000000000000000000..9815152f46e2fbec70f777c1f929f88b396bfab0 --- /dev/null +++ b/Venus/cluster_full_log.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd66bdd91f3aa7a8469e84aa7e8df886ed22b539daa45cae4f4baca3b624b198 +size 22943535 diff --git a/Venus/cluster_full_log_h.csv b/Venus/cluster_full_log_h.csv new file mode 100644 index 0000000000000000000000000000000000000000..69287ca5fc80f5ec0940f8a4d0979e964df2496c --- /dev/null +++ b/Venus/cluster_full_log_h.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3f3603ec611d62b00082adcfccb5b6e051f5866306fd8238729d59cffa6f932 +size 24130147 diff --git a/Venus/cluster_full_log_l.csv b/Venus/cluster_full_log_l.csv new file mode 100644 index 0000000000000000000000000000000000000000..bd52a3aba38641bab92fe3f9420a876c1e0c7c50 --- /dev/null +++ b/Venus/cluster_full_log_l.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96e8e8a5c4e3ff6f95b2c51ccc624ee04dd02515cffb4f53cc7b0fc50418d4e7 +size 24050975 diff --git a/Venus/cluster_log.csv b/Venus/cluster_log.csv new file mode 100644 index 0000000000000000000000000000000000000000..566bf2208759c0f5242385e15c269acc1b43a0ee --- /dev/null +++ b/Venus/cluster_log.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf5d90e653d2524de5cc26f796a1a6bbf577b20f6a10563badd9595bdfe11c5b +size 34465498 diff --git a/Venus/cluster_throughput.csv b/Venus/cluster_throughput.csv new file mode 100644 index 0000000000000000000000000000000000000000..9ec0136248169d5e3cc6b64167a85b18a39b9f98 --- /dev/null +++ b/Venus/cluster_throughput.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69642494d5f3cb9dff382de90ce1b2fac6c53f63c765e0c3eed9741cb20d4674 +size 1064423 diff --git a/Venus/vc_config.csv b/Venus/vc_config.csv new file mode 100644 index 0000000000000000000000000000000000000000..16a398ac11e17e6fdb21ae99385bfb5fd62d99e1 --- /dev/null +++ b/Venus/vc_config.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e25f63eecc9d19e905f55c37dceddffcd3a797bf83895fc7efe63c8cde6ca524 +size 130 diff --git a/colocate_info.csv b/colocate_info.csv new file mode 100644 index 0000000000000000000000000000000000000000..1e6717afc9cbed534d4a6a7d6acbb740e3560241 --- /dev/null +++ b/colocate_info.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f4954d7935a5e185450a162ec7787618cc43168b59a0e4fac03fef4108a62a0 +size 140553 diff --git a/colocate_info_old.csv b/colocate_info_old.csv new file mode 100644 index 0000000000000000000000000000000000000000..76bb559b89a88516a9a8bed466c0d30f7bb1d5cc --- /dev/null +++ b/colocate_info_old.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f7c6de4c3ebeea40a42f9c0e9c47aa483414e6fec8c1e6de89fb1b210f5c1dd +size 116094 diff --git a/colocate_info_real.csv b/colocate_info_real.csv new file mode 100644 index 0000000000000000000000000000000000000000..d14a6352605c0e0cd2880e079d7209ce1cf736c0 --- /dev/null +++ b/colocate_info_real.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb17384138b035112e7118b9fd33ae2ab9040fb4688942c74af59a700425ed0d +size 116099 diff --git a/colocate_info_square.csv b/colocate_info_square.csv new file mode 100644 index 0000000000000000000000000000000000000000..e5a352c0042f94c6b16c6752d6aa2198984f2161 --- /dev/null +++ b/colocate_info_square.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47acca3e7393b01678ce0f1fb0d800ee14cb43f762a7bb2dcb01ae7e58c0a2e8 +size 138165 diff --git a/dict2csv.py b/dict2csv.py new file mode 100644 index 0000000000000000000000000000000000000000..ef7de5ee5b21edb29a9206c5dfb1d7681daa14ec --- /dev/null +++ b/dict2csv.py @@ -0,0 +1,17 @@ +import pandas as pd +import pickle + +# df = pd.read_csv("philly_vc.csv") +# d = dict(zip(df["vc"].values, df["node num"].values)) + +# with open(f"./vc_dict_homo.pkl", "wb") as f: +# pickle.dump(d, f, pickle.HIGHEST_PROTOCOL) + +cluster_list = ["Venus", "Earth", "Saturn", "Uranus", "Philly"] + +for i, v in enumerate(cluster_list): + vc_dict = pd.read_pickle(v + "/vc_dict_homo.pkl") + df = pd.DataFrame.from_dict(vc_dict, orient="index", columns=["num"]) + df.to_csv(v + "/vc_config.csv") + +print(df.to_dict()["num"]) diff --git a/log_process.py b/log_process.py new file mode 100644 index 0000000000000000000000000000000000000000..5108334e198562366930b96240229f5384167e33 --- /dev/null +++ b/log_process.py @@ -0,0 +1,119 @@ +import argparse +import pandas as pd +from pathlib import Path + +""" +Processing log files for simulation +""" + + +def main(args): + cluster = args.cluster + + if not Path(f"./{cluster}").exists(): + Path(f"./{cluster}").mkdir() + + if cluster == "Philly": + logfile = Path(f"../../../analysis/1_compare with Philly trace/philly_trace.csv") + else: + logfile = Path(f"../../../data/{cluster}/cluster_log.csv") + + df = pd.read_csv(logfile, parse_dates=["submit_time", "start_time", "end_time"]) + df = df.sort_values(by="submit_time") + df.reset_index(drop=True, inplace=True) + + if cluster == "Neptune": + df = df[df["vc"] != "vc7Bz"] + df = df[df["vc"] != "vcIoD"] + df = df[df["vc"] != "vcftk"] + df = df[df["vc"] != "vc5LC"] + df = df[df["vc"] != "vcEwI"] + + df.loc[df["vc"] == "vcVvI", "vc"] = "vcUV3" + df.loc[df["vc"] == "vcrsE", "vc"] = "vcBUL" + df.loc[df["vc"] == "vcHyk", "vc"] = "vcBUL" + + df.reset_index(drop=True, inplace=True) + df.to_csv(f"./{cluster}/cluster_log.csv", index=False) + + elif cluster == "Saturn": + df = df[df["vc"] != "vc7Bz"] + df = df[df["vc"] != "vcHcQ"] + df = df[df["vc"] != "vck1d"] + df = df[df["vc"] != "vcj72"] + df = df[df["vc"] != "vcIya"] + df = df[df["vc"] != "vcygX"] + df = df[df["vc"] != "vcxqr"] + df = df[df["vc"] != "vcsgw"] + + df.reset_index(drop=True, inplace=True) + df.to_csv(f"./{cluster}/cluster_log.csv", index=False) + + elif cluster == "Uranus": + df = df[df["vc"] != "vc7Bz"] + df = df[df["vc"] != "vczGr"] + df = df[df["vc"] != "vciN1"] + df = df[df["vc"] != "vcV7h"] + df = df[df["vc"] != "vcRAl"] + df = df[df["vc"] != "vcvcM"] + df = df[df["vc"] != "vc1z2"] + + df.loc[df["vc"] == "vcVvI", "vc"] = "vcUV3" + df.loc[df["vc"] == "vcxqr", "vc"] = "vcUV3" + df.loc[df["vc"] == "vcsBT", "vc"] = "vcUV3" + df.loc[df["vc"] == "vcygX", "vc"] = "vcUV3" + df.loc[df["vc"] == "vcHyk", "vc"] = "vcOlr" + df.loc[df["vc"] == "vcRDh", "vc"] = "vc7hD" + df.loc[df["vc"] == "vcFsC", "vc"] = "vc7hD" + + df.reset_index(drop=True, inplace=True) + df.to_csv(f"./{cluster}/cluster_log.csv", index=False) + + elif cluster == "Earth": + df = df[df["vc"] != "vcp4O"] + df = df[df["vc"] != "vcvcM"] + df = df[df["vc"] != "vcXrB"] + df = df[df["vc"] != "vc7hD"] + df = df[df["vc"] != "vcIya"] + df = df[df["vc"] != "vc8Sj"] + df = df[df["vc"] != "vcLJZ"] + + df.loc[df["vc"] == "vcxS0", "vc"] = "vc3sl" + + df.reset_index(drop=True, inplace=True) + df.to_csv(f"./{cluster}/cluster_log.csv", index=False) + + elif cluster == "Venus": + df = df[df["vc"] != "vcEhP"] + df = df[df["vc"] != "vcIya"] + df = df[df["vc"] != "vcJLV"] + df = df[df["vc"] != "vcJkd"] + df = df[df["vc"] != "vcsBT"] + + df.loc[df["vc"] == "vcbIW", "vc"] = "vcvGl" + df.loc[df["vc"] == "vc6YE", "vc"] = "vcvGl" + df.loc[df["vc"] == "vcOhe", "vc"] = "vcKeu" + df.loc[df["vc"] == "vccJW", "vc"] = "vcKeu" + df.loc[df["vc"] == "vcP2J", "vc"] = "vchA3" + + df.reset_index(drop=True, inplace=True) + df.to_csv(f"./{cluster}/cluster_log.csv", index=False) + + elif cluster == "Philly": + df = df[df["vc"] != "795a4c"] + df = df[df["vc"] != "51b7ef"] + df = df[df["vc"] != "925e2b"] + df = df[df["vc"] != "23dbec"] + + df.reset_index(drop=True, inplace=True) + df.to_csv(f"./{cluster}/cluster_log.csv", index=False) + + else: + raise ValueError("Wrong Cluster Name.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Job Log Processor") + parser.add_argument("-c", "--cluster", default="Earth", type=str, help="Cluster Name") + args = parser.parse_args() + main(args) diff --git a/prepare_data.sh b/prepare_data.sh new file mode 100644 index 0000000000000000000000000000000000000000..d994a2de033a7000eaba05707dcce22cb7605a1f --- /dev/null +++ b/prepare_data.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +:< 0] + + # parse each job + for _, job in log.iterrows(): + gnum = job["gpu_num"] + job["submit_time"] = job["submit_time"].replace(second=0) + job["start_time"] = job["start_time"].replace(second=0) + job["end_time"] = job["end_time"].replace(second=0) + + run = (df["time"] >= job["start_time"]) & (df["time"] <= job["end_time"]) + + if gnum > 0: + df.loc[run, "running_gpujob_num"] += 1 + df.loc[run, "running_gpu_num"] += gnum + + if gnum > 1: + df.loc[run, "running_gpu_multi"] += gnum + else: + df.loc[run, "running_gpu_single"] += gnum + + cluster_gpu = cluster_gpu[["date", "total"]] + cluster_gpu = cluster_gpu[ + (cluster_gpu["date"] >= pd.Timestamp(date_range[0])) & (cluster_gpu["date"] <= pd.Timestamp(date_range[1])) + ] + cluster_gpu = pd.concat([cluster_gpu] * 1440).sort_index() + df["total_gpu_num"] = cluster_gpu["total"].values + + df["running_gpu_num"] = df["running_gpu_num"].combine(df["total_gpu_num"], min) + df["gpu_utilization"] = (df["running_gpu_num"] / df["total_gpu_num"]).round(3) + + df["gpu_utilization_multi"] = (df["running_gpu_multi"] / df["total_gpu_num"]).round(3) + df["gpu_utilization_single"] = (df["running_gpu_single"] / df["total_gpu_num"]).round(3) + df.set_index("time", inplace=True, drop=True) + return df + + +"""Script for generating cluster throughput file""" + + +def parse_throughput(dir): + date_range = ("2020-04-01 00:00:00", "2020-09-28 23:50:00") + log = pd.read_csv(f"{dir}/cluster_log.csv", parse_dates=["submit_time", "start_time", "end_time"]) + df = pd.DataFrame(pd.date_range(start=date_range[0], end=date_range[1], freq="10T"), columns=["time"],) + df[ + [ + "submit_job_all", + "start_job_all", + "end_job_all", + "submit_gpu_job", + "start_gpu_job", + "end_gpu_job", + "submit_gpu_num", + "start_gpu_num", + "end_gpu_num", + ] + ] = 0 + df.set_index("time", inplace=True, drop=True) + + for i in range(len(df)): + for kind in ("submit", "start", "end"): + jobs = log[(log[kind + "_time"] >= df.index[i]) & (log[kind + "_time"] < df.index[i] + timedelta(minutes=10))] + df[kind + "_job_all"][i] = len(jobs) + df[kind + "_gpu_job"][i] = len(jobs[jobs["gpu_num"] != 0]) + df[kind + "_gpu_num"][i] = jobs["gpu_num"].agg(sum) + return df + + +def parse_throughput_philly(dir): + date_range = ("2017-10-01 00:00:00", "2017-11-18 23:50:00") + log = pd.read_csv(f"{dir}/cluster_log.csv", parse_dates=["submit_time", "start_time", "end_time"]) + df = pd.DataFrame(pd.date_range(start=date_range[0], end=date_range[1], freq="10T"), columns=["time"],) + df[["submit_gpu_job", "start_gpu_job", "end_gpu_job", "submit_gpu_num", "start_gpu_num", "end_gpu_num"]] = 0 + df.set_index("time", inplace=True, drop=True) + + for i in range(len(df)): + for kind in ("submit", "start", "end"): + jobs = log[(log[kind + "_time"] >= df.index[i]) & (log[kind + "_time"] < df.index[i] + timedelta(minutes=10))] + # df.at[i, kind + "_gpu_job"] = len(jobs[jobs["gpu_num"] != 0]) + # df.at[i, kind + "_gpu_num"] = jobs["gpu_num"].agg(sum) + df[kind + "_gpu_job"][i] = len(jobs[jobs["gpu_num"] != 0]) + df[kind + "_gpu_num"][i] = jobs["gpu_num"].agg(sum) + return df + + +"""Script for generating cluster user file""" + + +def parse_user(dir, helios=False): + # df : contain cpu and gpu jobs + # dfgpu: only gpu jobs + # helios: analyze the whole data center users + if helios: + df = pd.read_csv(f"{dir}/all_cluster_log.csv", parse_dates=["submit_time", "start_time", "end_time"],) + else: + df = pd.read_csv(f"{dir}/cluster_log.csv", parse_dates=["submit_time", "start_time", "end_time"],) + + dfgpu = df[df["gpu_num"] > 0] + + users = df["user"].unique() + users_gpu = dfgpu["user"].unique() + + user_df = pd.DataFrame({"user": users}) + user_df = user_df.set_index("user") + user_df["cpu_only"] = False + user_df[["vc_list", "vc_list_gpu"]] = None + + for u in users: + data = df[df["user"] == u] + datagpu = dfgpu[dfgpu["user"] == u] + + if u in users_gpu: + cpu_job_num = len(data) - len(datagpu) + + plist = data["vc"].unique().tolist() + user_df.at[u, "vc_list"] = plist + user_df.at[u, "vc_num"] = len(plist) + + glist = datagpu["vc"].unique().tolist() + user_df.at[u, "vc_list_gpu"] = glist + user_df.at[u, "vc_num_gpu"] = len(glist) + + user_df.at[u, "job_num"] = len(data) + user_df.at[u, "gpu_job_num"] = len(datagpu) + user_df.at[u, "cpu_job_num"] = cpu_job_num + + user_df.at[u, "avg_run_time"] = data["duration"].mean() + user_df.at[u, "avg_gpu_run_time"] = datagpu["duration"].mean() + user_df.at[u, "avg_cpu_run_time"] = ( + 0 if cpu_job_num == 0 else (data["duration"].sum() - datagpu["duration"].sum()) / cpu_job_num + ) + + user_df.at[u, "avg_pend_time"] = data["queue"].mean() + user_df.at[u, "avg_gpu_pend_time"] = datagpu["queue"].mean() + user_df.at[u, "avg_cpu_pend_time"] = ( + 0 if cpu_job_num == 0 else (data["queue"].sum() - datagpu["queue"].sum()) / cpu_job_num + ) + + user_df.at[u, "avg_gpu_num"] = data["gpu_num"].mean() + user_df.at[u, "avg_cpu_num"] = data["cpu_num"].mean() + + user_df.at[u, "total_gpu_time"] = (datagpu["gpu_num"] * datagpu["duration"]).sum() + user_df.at[u, "total_cpu_time"] = (data["cpu_num"] * data["duration"]).sum() + user_df.at[u, "total_cpu_only_time"] = ( + user_df.at[u, "total_cpu_time"] - (datagpu["cpu_num"] * datagpu["duration"]).sum() + ) + user_df.at[u, "total_gpu_pend_time"] = datagpu["queue"].sum() + + user_df.at[u, "completed_percent"] = len(data[data["state"] == "COMPLETED"]) / len(data) + user_df.at[u, "completed_gpu_percent"] = len(datagpu[datagpu["state"] == "COMPLETED"]) / len(datagpu) + user_df.at[u, "completed_cpu_percent"] = ( + 0 + if cpu_job_num == 0 + else (len(data[data["state"] == "COMPLETED"]) - len(datagpu[datagpu["state"] == "COMPLETED"])) / cpu_job_num + ) + + user_df.at[u, "cencelled_percent"] = len(data[data["state"] == "CANCELLED"]) / len(data) + user_df.at[u, "cencelled_gpu_percent"] = len(datagpu[datagpu["state"] == "CANCELLED"]) / len(datagpu) + user_df.at[u, "cencelled_cpu_percent"] = ( + 0 + if cpu_job_num == 0 + else (len(data[data["state"] == "CANCELLED"]) - len(datagpu[datagpu["state"] == "CANCELLED"])) / cpu_job_num + ) + + user_df.at[u, "failed_percent"] = len(data[data["state"] == "FAILED"]) / len(data) + user_df.at[u, "failed_gpu_percent"] = len(datagpu[datagpu["state"] == "FAILED"]) / len(datagpu) + user_df.at[u, "failed_cpu_percent"] = ( + 0 + if cpu_job_num == 0 + else (len(data[data["state"] == "FAILED"]) - len(datagpu[datagpu["state"] == "FAILED"])) / cpu_job_num + ) + else: + user_df.at[u, "cpu_only"] = True + + plist = data["vc"].unique().tolist() + user_df.at[u, "vc_list"] = plist + user_df.at[u, "vc_num"] = len(plist) + + user_df.at[u, "vc_list_gpu"] = [] + user_df.at[u, "vc_num_gpu"] = 0 + + user_df.at[u, "job_num"] = len(data) + user_df.at[u, "gpu_job_num"] = 0 + user_df.at[u, "cpu_job_num"] = len(data) + + user_df.at[u, "avg_run_time"] = data["duration"].mean() + user_df.at[u, "avg_gpu_run_time"] = 0 + user_df.at[u, "avg_cpu_run_time"] = user_df.at[u, "avg_run_time"] + + user_df.at[u, "avg_pend_time"] = data["queue"].mean() + user_df.at[u, "avg_gpu_pend_time"] = 0 + user_df.at[u, "avg_cpu_pend_time"] = user_df.at[u, "avg_pend_time"] + + user_df.at[u, "avg_gpu_num"] = 0 + user_df.at[u, "avg_cpu_num"] = data["cpu_num"].mean() + + user_df.at[u, "total_gpu_time"] = 0 + user_df.at[u, "total_cpu_time"] = (data["cpu_num"] * data["duration"]).sum() + user_df.at[u, "total_cpu_only_time"] = user_df.at[u, "total_cpu_time"] + user_df.at[u, "total_gpu_pend_time"] = 0 + + user_df.at[u, "completed_percent"] = len(data[data["state"] == "COMPLETED"]) / len(data) + user_df.at[u, "completed_gpu_percent"] = 0 + user_df.at[u, "completed_cpu_percent"] = user_df.at[u, "completed_percent"] + + user_df.at[u, "cencelled_percent"] = len(data[data["state"] == "CANCELLED"]) / len(data) + user_df.at[u, "cencelled_gpu_percent"] = 0 + user_df.at[u, "cencelled_cpu_percent"] = user_df.at[u, "cencelled_percent"] + + user_df.at[u, "failed_percent"] = len(data[data["state"] == "FAILED"]) / len(data) + user_df.at[u, "failed_gpu_percent"] = 0 + user_df.at[u, "failed_cpu_percent"] = user_df.at[u, "failed_percent"] + + user_df.sort_values(by="job_num", ascending=False, inplace=True) + user_df[["vc_num", "vc_num_gpu", "job_num", "gpu_job_num", "cpu_job_num"]] = user_df[ + ["vc_num", "vc_num_gpu", "job_num", "gpu_job_num", "cpu_job_num"] + ].astype(int) + return user_df + + +"""Script for generating a all cluster trace file""" + + +def parse_all_cluster_log(clusters): + logall = pd.DataFrame() + + for cluster in clusters: + print(f"Parsing {cluster}") + data_dir = f"../data/{cluster}" + log = pd.read_csv(f"{data_dir}/cluster_log.csv", parse_dates=["submit_time", "start_time", "end_time"]) + log["c"] = cluster + logall = pd.concat([logall, log]) + + logall.insert(1, "cluster", logall["c"]) + logall.drop(columns="c", inplace=True) + logall.to_csv("../data/all_cluster_log.csv", index=False) + + +"""Script for generating all cluster monthly file""" + + +def parse_monthly_job(cluster_name): + df = pd.DataFrame(cluster_name, columns=["id"]) + df.set_index("id", drop=True, inplace=True) + + for cluster in cluster_name: + data_dir = f"../data/{cluster}" + log = pd.read_csv(f"{data_dir}/cluster_log.csv", parse_dates=["submit_time", "start_time", "end_time"]) + log["month"] = getattr(log["submit_time"].dt, "month").astype(np.int16) + glog = log[log["gpu_num"] > 0] + clog = log[log["gpu_num"] == 0] + for m in range(4, 10): + month_glog = glog[glog["month"] == m] + df.at[cluster, str(m) + " Job Num"] = len(log[log["month"] == m]) + df.at[cluster, str(m) + " CPU Job Num"] = len(clog[clog["month"] == m]) + df.at[cluster, str(m) + " GPU Job Num"] = len(month_glog) + df.at[cluster, str(m) + " GJobNum 1"] = len(month_glog[month_glog["gpu_num"] == 1]) + df.at[cluster, str(m) + " GJobNum g1"] = len(month_glog[month_glog["gpu_num"] > 1]) + + df = df.astype(int) + df.to_csv("../data/all_cluster_monthly_job_num.csv") + + +def parse_monthly_util(cluster_list): + df = pd.DataFrame() + + for i in range(len(cluster_list)): + seq = pd.read_csv(f"../data/{cluster_list[i]}/cluster_sequence.csv", parse_dates=["time"], index_col="time",) + seq["month"] = seq.index.month + df.loc[:, cluster_list[i]] = seq.groupby("month").mean()["gpu_utilization"].values + + df.to_csv("../data/all_cluster_monthly_utilization.csv", index=False) + + +if __name__ == "__main__": + # parser = argparse.ArgumentParser(description="Trace Parser") + # parser.add_argument("-c", "--cluster-list", default=["Venus", "Earth", "Saturn", "Uranus"], help="Cluster list for parsing") + # args = parser.parse_args() + + # clusters = args.cluster_list + # if type(clusters) is not list: + # clusters = [clusters] + + # print("1/4. Generating all cluster log file.") + # parse_all_cluster_log(clusters) + + # print("2/4. Generating cluster sequence and throughput files. This step may take hours.") + # for cluster in clusters: + # print(f"Parsing {cluster}") + # data_dir = f"../data/{cluster}" + # # seq = parse_sequence(data_dir) + # df = parse_throughput(data_dir) + + # # seq.to_csv(f"{data_dir}/cluster_sequence.csv") + # df.to_csv(f"{data_dir}/cluster_throughput.csv") + + # print("3/4. Generating monthly job num and utilization files") + # parse_monthly_job(clusters) + # parse_monthly_util(clusters) + + # print("4/4. Generating monthly job num and utilization files") + # data_dir = f"../data" + # df = parse_user(data_dir, helios=True) + # df.to_pickle(f"{data_dir}/cluster_user.pkl") + + # for cluster in clusters: + # print(f"Parsing {cluster}") + # data_dir = f"../data/{cluster}" + # df = parse_user(data_dir) + + # df.to_pickle(f"{data_dir}/cluster_user.pkl") + + print("Philly Throughput. This step may take hours.") + data_dir = f"./Philly" + # seq = parse_sequence(data_dir) + df = parse_throughput_philly(data_dir) + + # seq.to_csv(f"{data_dir}/cluster_sequence.csv") + df.to_csv(f"{data_dir}/cluster_throughput.csv") diff --git a/vc_dict_generator.py b/vc_dict_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..2f91614509141101745e5b8ec4acf8b97a535fa9 --- /dev/null +++ b/vc_dict_generator.py @@ -0,0 +1,165 @@ +import argparse +import pandas as pd +from pathlib import Path +import datetime +import pickle + +""" +Generate VC configuration for simulation +""" + + +def main(args): + cluster = args.cluster + date = args.date + + if not Path(f"./{cluster}").exists(): + Path(f"./{cluster}").mkdir() + + if cluster == "Philly": + df = pd.read_csv(Path(f"./philly_vc.csv")) + else: + file = Path(f"../../../data/{cluster}/cluster_gpu_number.csv") + df = pd.read_csv(file, parse_dates=["date"]) + + if cluster == "Jupiter" and date == "July": + vcs = df.columns.values[1:-2] + nodes_num = df[df["date"] == datetime.datetime(2020, 7, 1)].values[0][1:-2] + nodes_num = nodes_num // 8 + + assert len(nodes_num) == len(vcs) + dic = dict(zip(vcs, nodes_num)) + del dic["vc7Bz"] # Not Consider Test VC + + with open(f"./{cluster}/vc_dict.pkl", "wb") as f: + pickle.dump(dic, f, pickle.HIGHEST_PROTOCOL) + + if cluster == "Neptune" and date == "Sept": + vcs = df.columns.values[1:-1] + nodes_num = df[df["date"] == datetime.datetime(2020, 9, 1)].values[0][1:-1] + nodes_num = nodes_num // 8 + + assert len(nodes_num) == len(vcs) + dic = dict(zip(vcs, nodes_num)) + + del dic["vc7Bz"] # Not Consider Test VC + del dic["vcIoD"] # Zero nodes + del dic["vcftk"] # Zero nodes + del dic["vc5LC"] # No Job + del dic["vcEwI"] # No Job + + # Some vc rename / merge together + del dic["vcrsE"] # Merge to vcBUL + del dic["vcHyk"] # Merge to vcBUL + del dic["vcVvI"] # Merge to vcUV3 + + with open(f"./{cluster}/vc_dict.pkl", "wb") as f: + pickle.dump(dic, f, pickle.HIGHEST_PROTOCOL) + + if cluster == "Saturn" and date == "Sept": + vcs = df.columns.values[1:-1] + nodes_num = df[df["date"] == datetime.datetime(2020, 9, 1)].values[0][1:-1] + nodes_num = nodes_num // 8 + + assert len(nodes_num) == len(vcs) + dic = dict(zip(vcs, nodes_num)) + + del dic["vc7Bz"] # Not Consider Test VC + del dic["vcHcQ"] # Zero nodes + del dic["vck1d"] # Zero nodes + del dic["vcj72"] # Zero nodes + del dic["vcIya"] # Zero nodes + del dic["vcygX"] # Zero nodes + del dic["vcxqr"] # Zero nodes + del dic["vcsgw"] # Zero nodes + + with open(f"./{cluster}/vc_dict.pkl", "wb") as f: + pickle.dump(dic, f, pickle.HIGHEST_PROTOCOL) + + if cluster == "Uranus" and date == "Sept": + vcs = df.columns.values[1:-1] + nodes_num = df[df["date"] == datetime.datetime(2020, 9, 10)].values[0][1:-1] + nodes_num = nodes_num // 8 + + assert len(nodes_num) == len(vcs) + dic = dict(zip(vcs, nodes_num)) + + del dic["vc7Bz"] # Not Consider Test VC + del dic["vczGr"] # Zero nodes + del dic["vciN1"] # Zero nodes + del dic["vcV7h"] # Zero nodes + del dic["vcRAl"] # Zero nodes + del dic["vcvcM"] # Zero nodes + del dic["vc1z2"] # Zero nodes + + del dic["vcHyk"] # Merge to vcOlr + del dic["vcRDh"] # Merge to vc7hD + del dic["vcFsC"] # Merge to vc7hD + del dic["vcVvI"] # Merge to vcUV3 + del dic["vcxqr"] # Merge to vcUV3 + del dic["vcsBT"] # Merge to vcUV3 + del dic["vcygX"] # Merge to vcUV3 + + with open(f"./{cluster}/vc_dict.pkl", "wb") as f: + pickle.dump(dic, f, pickle.HIGHEST_PROTOCOL) + + if cluster == "Earth" and date == "Sept": + vcs = df.columns.values[1:-1] + nodes_num = df[df["date"] == datetime.datetime(2020, 9, 10)].values[0][1:-1] + nodes_num = nodes_num // 8 + + assert len(nodes_num) == len(vcs) + dic = dict(zip(vcs, nodes_num)) + + # del dic['vc7Bz'] # Not Consider Test VC + del dic["vcp4O"] # Zero nodes + del dic["vcvcM"] # Zero nodes + del dic["vcXrB"] # Zero nodes + del dic["vc7hD"] # Zero nodes + del dic["vcIya"] # Zero nodes + del dic["vc8Sj"] # Zero nodes + + del dic["vcLJZ"] # Zero nodes in Sept + + del dic["vcxS0"] # Merge to vc3sl + + with open(f"./{cluster}/vc_dict.pkl", "wb") as f: + pickle.dump(dic, f, pickle.HIGHEST_PROTOCOL) + + if cluster == "Venus" and date == "Sept": + vcs = df.columns.values[1:-1] + nodes_num = df[df["date"] == datetime.datetime(2020, 9, 10)].values[0][1:-1] + nodes_num = nodes_num // 8 + + assert len(nodes_num) == len(vcs) + dic = dict(zip(vcs, nodes_num)) + + # del dic['vc7Bz'] # Not Consider Test VC + del dic["vcEhP"] # Zero nodes + del dic["vcIya"] # Zero nodes + del dic["vcJLV"] # Zero nodes + del dic["vcJkd"] # Zero nodes + del dic["vcsBT"] # Zero nodes + + del dic["vcbIW"] # Merge to vcvGl + del dic["vc6YE"] # Merge to vcvGl + del dic["vcOhe"] # Merge to vcKeu + del dic["vccJW"] # Merge to vcKeu + del dic["vcP2J"] # Merge to vchA3 + + with open(f"./{cluster}/vc_dict.pkl", "wb") as f: + pickle.dump(dic, f, pickle.HIGHEST_PROTOCOL) + + if cluster == "Philly": + d = dict(zip(df["vc"].values, df["node num"].values)) + + with open(f"./Philly/vc_dict_homo.pkl", "wb") as f: + pickle.dump(d, f, pickle.HIGHEST_PROTOCOL) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="VC Configuration Generator") + parser.add_argument("-c", "--cluster", default="Earth", type=str, help="Cluster Name") + parser.add_argument("-d", "--date", default="Sept", type=str, help="Month") + args = parser.parse_args() + main(args)