Skip to content

Commit 65ba2dc

Browse files
authored
tests: Refactor downloading of pannotia tests (gem5#1653)
With this patch the pannotia tests now: 1. Download the resources to 'gpu-pannotia' in the 'tests/gem5/resources' directory. This is where other test resources are store. 2. Download thr USA-road-d.NY.gr dataset from Google cloud bucket in a decompressed state. 2. Avoid re-download the resources if they are already present on the host machine.
1 parent 6195b33 commit 65ba2dc

File tree

1 file changed

+9
-7
lines changed

1 file changed

+9
-7
lines changed

tests/gem5/gpu/test_gpu_pannotia.py

+9-7
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,14 @@
2727
import gzip
2828
import os.path
2929
import shutil
30+
from pathlib import Path
3031
from urllib.request import urlretrieve
3132

3233
from testlib import *
3334

34-
resource_path = joinpath(absdirpath(__file__), "..", "gpu-pannotia-resources")
35+
resource_path = joinpath(
36+
absdirpath(__file__), "..", "resources", "gpu-pannotia"
37+
)
3538
binary_path = joinpath(resource_path, "pannotia-bins")
3639
dataset_path = joinpath(resource_path, "pannotia-datasets")
3740

@@ -57,7 +60,7 @@
5760
"G3_circuit.graph": "https://storage.googleapis.com/dist.gem5.org/dist/develop/datasets/pannotia/mis/G3_circuit.graph",
5861
"ecology1.graph": "https://storage.googleapis.com/dist.gem5.org/dist/develop/datasets/pannotia/mis/ecology1.graph",
5962
"coAuthorsDBLP.graph": "https://storage.googleapis.com/dist.gem5.org/dist/develop/datasets/pannotia/pagerank/coAuthorsDBLP.graph",
60-
"USA-road-d.NY.gr.gz": "http://www.diag.uniroma1.it/challenge9/data/USA-road-d/USA-road-d.NY.gr.gz",
63+
"USA-road-d.NY.gr": "https://storage.googleapis.com/dist.gem5.org/dist/develop/datasets/pannotia/USA-road-d/USA-road-d.NY.gr",
6164
}
6265

6366

@@ -66,15 +69,14 @@
6669
os.makedirs(dataset_path)
6770

6871
for name in binary_links.keys():
72+
if Path(f"{binary_path}/{name}").exists():
73+
continue
6974
urlretrieve(binary_links[name], f"{binary_path}/{name}")
7075
for name in dataset_links.keys():
76+
if Path(f"{dataset_path}/{name}").exists():
77+
continue
7178
urlretrieve(dataset_links[name], f"{dataset_path}/{name}")
7279

73-
with gzip.open(f"{dataset_path}/USA-road-d.NY.gr.gz", "rb") as f_in:
74-
with open(f"{dataset_path}/USA-road-d.NY.gr", "wb") as f_out:
75-
shutil.copyfileobj(f_in, f_out)
76-
os.remove(f"{dataset_path}/USA-road-d.NY.gr.gz")
77-
7880
if len(os.listdir(binary_path)) < len(binary_links):
7981
testlib.log.test_log.warn(
8082
"One or more binaries for the Pannotia GPU tests are missing! Try deleting gpu-pannotia-resources and rerunning."

0 commit comments

Comments
 (0)