Skip to content

Commit debfc17

Browse files
twsearletwsearle
authored
Merge branch 'develop' into feature/update-atlas-0.38.1
2 parents 80005e6 + 72dd763 commit debfc17

File tree

2 files changed

+372
-0
lines changed

2 files changed

+372
-0
lines changed

‎.github/workflows/build_ci_image.yml

+85
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
name: Create and publish the CI docker image
2+
3+
# Configures this workflow to run when triggered.
4+
on:
5+
push:
6+
tags:
7+
- v**
8+
workflow_dispatch:
9+
10+
# Defines two custom environment variables for the workflow. These are used for
11+
# the Container registry domain, and a name for the Docker image that this
12+
# workflow builds.
13+
env:
14+
REGISTRY: ghcr.io
15+
IMAGE_NAME: darth/orca-jedi/ci-almalinux9
16+
17+
jobs:
18+
build-and-push-image:
19+
runs-on: ubuntu-latest
20+
# Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
21+
permissions:
22+
contents: read
23+
packages: write
24+
attestations: write
25+
id-token: write
26+
#
27+
steps:
28+
- name: Checkout repository
29+
uses: actions/checkout@v4
30+
# Uses the `docker/login-action` action to log in to the Container
31+
# registry registry using the account and password that will publish the
32+
# packages. Once published, the packages are scoped to the account
33+
# defined here.
34+
- name: Log in to the Container registry
35+
uses: docker/login-action@v3
36+
with:
37+
registry: ${{ env.REGISTRY }}
38+
username: ${{ github.actor }}
39+
password: ${{ secrets.GH_PAT }}
40+
# This step uses
41+
# [docker/metadata-action](https://github.com/docker/metadata-action#about)
42+
# to extract tags and labels that will be applied to the specified image.
43+
# The `id` "meta" allows the output of this step to be referenced in a
44+
# subsequent step. The `images` value provides the base name for the tags
45+
# and labels.
46+
- name: Extract metadata (tags, labels) for Docker
47+
id: meta
48+
uses: docker/metadata-action@v5
49+
with:
50+
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
51+
- name: Install hpccm
52+
run: |
53+
python3 -m pip install hpccm
54+
- name: Recipe to Dockerfile
55+
run: |
56+
hpccm --recipe ./ci/hpccm_recipe_almalinux9.py >./Dockerfile
57+
# This step uses the `docker/build-push-action` action to build the
58+
# image, based on your repository's `Dockerfile`. If the build succeeds,
59+
# it pushes the image to GitHub Packages. It uses the `context`
60+
# parameter to define the build's context as the set of files located in
61+
# the specified path. For more information, see
62+
# "[Usage](https://github.com/docker/build-push-action#usage)" in the
63+
# README of the `docker/build-push-action` repository. It uses the
64+
# `tags` and `labels` parameters to tag and label the image with the
65+
# output from the "meta" step.
66+
- name: Build and push Docker image
67+
id: push
68+
uses: docker/build-push-action@v6
69+
with:
70+
context: .
71+
push: true
72+
tags: ${{ steps.meta.outputs.tags }}
73+
labels: ${{ steps.meta.outputs.labels }}
74+
75+
# This step generates an artifact attestation for the image, which is an
76+
# unforgeable statement about where and how it was built. It increases
77+
# supply chain security for people who consume the image. For more
78+
# information, see
79+
# "[AUTOTITLE](/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds)."
80+
- name: Generate artifact attestation
81+
uses: actions/attest-build-provenance@v1
82+
with:
83+
subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
84+
subject-digest: ${{ steps.push.outputs.digest }}
85+
push-to-registry: true

‎ci/hpccm_recipe_almalinux9.py

+287
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,287 @@
1+
# © Copyright 2021 Met Office
2+
# This software is licensed under the terms of the Apache Licence Version 2.0 which can be obtained at
3+
# http://www.apache.org/licenses/LICENSE-2.0.
4+
#
5+
6+
"""JOPA development container (simplified)
7+
8+
Usage:
9+
hpccm --recipe THIS-FILE [options] > DEST
10+
"""
11+
12+
13+
def github_url(repo, vn):
14+
return f'https://github.com/{repo}/archive/refs/tags/{vn}.tar.gz'
15+
16+
17+
def gitlab_url(repo, vn):
18+
name = repo.rsplit('/', 1)[1]
19+
return f'https://gitlab.com/{repo}/-/archive/{vn}/{name}-{vn}.tar.gz'
20+
21+
22+
# get versions via --userarg options
23+
# build hdf5/netcdf with zstd (libzstd from epel)
24+
atlas_orca_vn = USERARG.get('atlas_orca_vn', '0.3.1')
25+
atlas_vn = USERARG.get('atlas_vn', '0.37.0')
26+
blitz_vn = USERARG.get('blitz_vn', '1.0.2')
27+
boost_vn = USERARG.get('boost_vn', '1.85.0')
28+
bufr_query_vn = USERARG.get('bufr_query_vn', 'v0.0.1')
29+
cmake_vn = USERARG.get('cmake_vn', '3.30.0')
30+
ecbuild_vn = USERARG.get('ecbuild_vn', '3.8.5')
31+
eccodes_vn = USERARG.get('eccodes_vn', '2.30.1') # requires AEC (libaec-devel) by default
32+
eckit_vn = USERARG.get('eckit_vn', '1.26.2')
33+
ectrans_vn = USERARG.get('ectrans_vn', '1.2.0')
34+
fckit_vn = USERARG.get('fckit_vn', '0.13.0')
35+
fiat_vn = USERARG.get('fiat_vn', '1.4.1')
36+
fparser_vn = USERARG.get('fparser_vn', '0.1.4')
37+
gsl_lite_vn = USERARG.get('gsl_lite_vn', '0.41.0')
38+
gsw_fortran_vn = USERARG.get('gsw_fortran_vn', '3.07')
39+
hdf5_vn = USERARG.get('hdf5_vn', 'hdf5-1_14_2')
40+
json_schema_validator_vn = USERARG.get('json_schema_validator_vn', '2.3.0')
41+
json_vn = USERARG.get('json_vn', '3.9.1')
42+
lapack_vn = USERARG.get('lapack_vn', '3.11.0')
43+
nccmp_vn = USERARG.get('nccmp_vn', '1.9.1.0')
44+
nceplibs_bufr_vn = USERARG.get('nceplibs_bufr_vn', '12.0.1')
45+
netcdf_vn = USERARG.get('netcdf_vn', '4.9.2')
46+
netcdfcxx_vn = USERARG.get('netcdfcxx_vn', '4.3.1')
47+
netcdfftn_vn = USERARG.get('netcdfftn_vn', '4.6.1')
48+
odc_vn = USERARG.get('odc_vn', '1.5.2')
49+
openmpi_vn = USERARG.get('openmpi_vn', '4.1.5')
50+
pycodestyle_vn = USERARG.get('pycodestyle_vn', '2.10')
51+
udunits_vn = USERARG.get('udunits_vn', '2.2.28')
52+
yaxt_vn = USERARG.get('yaxt_vn', '528-0.10.0') # URL has a number and a version
53+
54+
COMMON_PACKAGES = [
55+
'bison',
56+
'bzip2',
57+
'clang-tools-extra',
58+
'eigen3-devel',
59+
'expat-devel',
60+
'flex',
61+
'gcc',
62+
'gcc-c++',
63+
'gcc-gfortran',
64+
'git',
65+
'git-lfs',
66+
'gmp-devel',
67+
'gnupg2',
68+
'graphviz',
69+
'jq',
70+
'lcov',
71+
'less',
72+
'libaec-devel',
73+
'libcurl-devel',
74+
'libX11-devel',
75+
'libxml2-devel',
76+
'libzstd-devel',
77+
'lz4-devel',
78+
'mpfr-devel',
79+
'ncurses-devel',
80+
'ninja-build',
81+
'openssh-server',
82+
'openssl-devel',
83+
'patch',
84+
'pkgconfig',
85+
'pybind11-devel',
86+
'python3-devel',
87+
'python3-pip',
88+
'python3-pytest',
89+
'python3-pyyaml',
90+
'rsync',
91+
'time',
92+
'unzip',
93+
'vim-minimal',
94+
'wget',
95+
'xz',
96+
'zlib-devel',
97+
'zstd',
98+
]
99+
100+
Stage0 += baseimage(image='almalinux:9', _as='build', _distro='rhel')
101+
Stage0 += shell(commands=[
102+
'dnf install -y \'dnf-command(config-manager)\'',
103+
'dnf config-manager -y --set-enabled crb',
104+
])
105+
Stage0 += packages(epel=True, ospackages=COMMON_PACKAGES)
106+
107+
Stage0 += cmake(eula=True, version=cmake_vn)
108+
109+
Stage0 += generic_cmake(
110+
prefix='/usr/local',
111+
url=github_url('Reference-LAPACK/lapack', f'v{lapack_vn}'),
112+
directory=f'lapack-{lapack_vn}',
113+
cmake_opts=['-DCMAKE_BUILD_TYPE=Release', '-DBUILD_SHARED_LIBS=ON'],
114+
)
115+
116+
Stage0 += boost(
117+
prefix='/usr/local',
118+
version=boost_vn,
119+
b2_opts=['toolset=gcc', 'cxxflags="-std=c++17"'],
120+
bootstrap_opts=[
121+
'--with-libraries=chrono,date_time,filesystem,program_options,regex,serialization,system,thread',
122+
],
123+
)
124+
125+
mpi = openmpi(
126+
prefix='/usr/local',
127+
version=openmpi_vn,
128+
cuda=False,
129+
infiniband=False,
130+
configure_opts=['--enable-mpi-fortran', '--enable-mpi-cxx'],
131+
)
132+
Stage0 += mpi
133+
134+
Stage0 += generic_autotools(
135+
prefix='/usr/local',
136+
repository='https://github.com/HDFGroup/hdf5',
137+
commit=hdf5_vn,
138+
enable_build_mode='production',
139+
enable_cxx=True,
140+
enable_fortran=True,
141+
enable_parallel=True,
142+
enable_threadsafe=True,
143+
enable_unsupported=True,
144+
toolchain=mpi.toolchain,
145+
with_szlib='/usr/local',
146+
with_zlib='/usr/local',
147+
)
148+
# - Post-process h5pcc to avoid f951: Warning:
149+
# Nonexistent include directory '/var/tmp/hdf5-1.14.0/src/H5FDsubfiling'
150+
# sed -i "s|-I${TMPDIR}/hdf5-${hdf5_vn}/src/H5FDsubfiling||g" "/usr/local/bin/h5pcc"
151+
Stage0 += shell(commands=[
152+
f'sed -i "s|-I/var/tmp/hdf5/src/H5FDsubfiling||g" "/usr/local/bin/h5pcc"'
153+
])
154+
Stage0 += environment(variables={'H5DIR': '/usr/local', 'LIBS': '-ldl'})
155+
Stage0 += netcdf(
156+
version=netcdf_vn,
157+
version_cxx=netcdfcxx_vn,
158+
version_fortran=netcdfftn_vn,
159+
prefix='/usr/local',
160+
cxx=True,
161+
fortran=True,
162+
enable_netcdf_4=True,
163+
enable_shared=True,
164+
disable_zstandard_plugin=True,
165+
toolchain=mpi.toolchain,
166+
)
167+
Stage0 += environment(variables={
168+
'NETCDF_DIR': '/usr/local',
169+
'NetCDF_ROOT': '/usr/local'})
170+
Stage0 += generic_cmake(
171+
prefix='/usr/local',
172+
url=gitlab_url('remikz/nccmp', nccmp_vn),
173+
cmake_opts=['-DCMAKE_BUILD_TYPE=Release', '-DBUILD_SHARED_LIBS=ON'],
174+
toolchain=mpi.toolchain,
175+
)
176+
177+
Stage0 += generic_autotools(
178+
prefix='/usr/local',
179+
url=f'https://downloads.unidata.ucar.edu/udunits/{udunits_vn}/udunits-{udunits_vn}.tar.gz',
180+
configure_opts=['--enable-shared=yes'],
181+
)
182+
183+
Stage0 += generic_cmake(
184+
prefix='/usr/local',
185+
url=github_url('gsl-lite/gsl-lite', f'v{gsl_lite_vn}'),
186+
directory=f'gsl-lite-{gsl_lite_vn}',
187+
cmake_opts=['-DCMAKE_BUILD_TYPE=Release'],
188+
)
189+
190+
Stage0 += generic_cmake(
191+
prefix='/usr/local',
192+
url=github_url('blitzpp/blitz', blitz_vn),
193+
directory=f'blitz-{blitz_vn}',
194+
cmake_opts=['-DCMAKE_BUILD_TYPE=Release', '-DBUILD_SHARED_LIBS=ON'],
195+
)
196+
197+
Stage0 += generic_cmake(
198+
prefix='/usr/local',
199+
url=github_url('nlohmann/json', f'v{json_vn}'),
200+
directory=f'json-{json_vn}',
201+
cmake_opts=['-DCMAKE_BUILD_TYPE=Release', '-DJSON_BuildTests=OFF'],
202+
)
203+
204+
Stage0 += generic_cmake(
205+
prefix='/usr/local',
206+
url=github_url('NOAA-EMC/NCEPLIBS-bufr', f'v{nceplibs_bufr_vn}'),
207+
directory=f'NCEPLIBS-bufr-{nceplibs_bufr_vn}',
208+
cmake_opts=['-DCMAKE_BUILD_TYPE=Release', '-DBUILD_TESTS=OFF'],
209+
)
210+
211+
Stage0 += generic_cmake(
212+
prefix='/usr/local',
213+
url=github_url('pboettch/json-schema-validator', json_schema_validator_vn),
214+
directory=f'json-schema-validator-{json_schema_validator_vn}',
215+
cmake_opts=[
216+
'-DCMAKE_BUILD_TYPE=Release',
217+
'-DCMAKE_POLICY_DEFAULT_CMP0074=NEW',
218+
'-DBUILD_SHARED_LIBS=ON',
219+
'-DBUILD_TESTS=OFF',
220+
'-DBUILD_EXAMPLES=OFF',
221+
],
222+
)
223+
224+
for org, name, vn in (
225+
('ecmwf', 'ecbuild', ecbuild_vn),
226+
('ecmwf', 'eckit', eckit_vn),
227+
('ecmwf', 'fckit', fckit_vn),
228+
('ecmwf', 'odc', odc_vn),
229+
('ecmwf-ifs', 'fiat', fiat_vn),
230+
('ecmwf-ifs', 'ectrans', ectrans_vn),
231+
('ecmwf', 'atlas', atlas_vn),
232+
('ecmwf', 'atlas-orca', atlas_orca_vn),
233+
('ecmwf', 'eccodes', eccodes_vn),
234+
):
235+
Stage0 += generic_cmake(
236+
prefix='/usr/local',
237+
url=github_url(f'{org}/{name}', vn),
238+
directory=f'{name}-{vn}',
239+
cmake_opts=['-DCMAKE_BUILD_TYPE=Release', '-DMPI=ON', '-DOMP=ON'],
240+
)
241+
242+
# -- bufr-query
243+
Stage0 += generic_cmake(
244+
prefix='/usr/local',
245+
repository='https://github.com/NOAA-EMC/bufr-query',
246+
commit=bufr_query_vn,
247+
cmake_opts=['-DCMAKE_BUILD_TYPE=Release', '-DMPI=ON', '-DOMP=ON'],
248+
build_environment={'LDFLAGS': '-lnetcdf'},
249+
)
250+
251+
# -- GSW-Fortran
252+
Stage0 += generic_cmake(
253+
prefix='/usr/local',
254+
url=github_url('JCSDA-internal/GSW-Fortran', f'v{gsw_fortran_vn}'),
255+
directory=f'GSW-Fortran-{gsw_fortran_vn}',
256+
cmake_opts=['-DCMAKE_BUILD_TYPE=Release', '-DMPI=ON', '-DOMP=ON'],
257+
)
258+
259+
yaxt_vns = yaxt_vn.split('-', 1)
260+
Stage0 += generic_autotools(
261+
prefix='/usr/local',
262+
url=(
263+
'https://swprojects.dkrz.de/redmine/attachments/download/' +
264+
f'{yaxt_vns[0]}/yaxt-{yaxt_vns[1]}.tar.xz'
265+
),
266+
configure_opts=['--with-idxtype=long', '--without-regard-for-quality'],
267+
)
268+
269+
Stage1 += baseimage(image='almalinux:9', _distro='rhel')
270+
Stage1 += comment('JEDI development image with GNU and OpenMPI')
271+
Stage1 += label(metadata={
272+
'Maintainer': '[email protected]',
273+
'Species': 'NextGen',
274+
'Version': 'v0.1'})
275+
Stage1 += shell(commands=[
276+
'dnf install -y \'dnf-command(config-manager)\'',
277+
'dnf config-manager -y --set-enabled crb',
278+
])
279+
Stage1 += packages(epel=True, ospackages=COMMON_PACKAGES)
280+
Stage1 += pip(pip='pip3', packages=['cpplint', 'yamlprocessor'])
281+
Stage1 += copy(_from='build', src='/usr/local', dest='/usr/local')
282+
Stage1 += shell(commands=['ln -sfT python3 /usr/bin/python'])
283+
Stage1 += environment(variables={
284+
'PATH': '/usr/local/bin:$PATH',
285+
'LD_LIBRARY_PATH': '/usr/local/lib64:/usr/local/lib:/usr/lib64:/usr/lib:$LD_LIBRARY_PATH',
286+
})
287+
Stage1 += workdir(directory='/var/tmp')

0 commit comments

Comments
 (0)