forked from mariadb-corporation/mariadb-columnstore-engine
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy path.drone.jsonnet
494 lines (465 loc) · 29.3 KB
/
.drone.jsonnet
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
local events = ['pull_request', 'cron'];
local platforms = {
develop: ['centos:7', 'rockylinux:8', 'debian:10', 'ubuntu:20.04'],
'develop-6': ['centos:7', 'rockylinux:8', 'debian:10', 'ubuntu:20.04'],
'develop-5': ['centos:7', 'rockylinux:8', 'debian:10', 'ubuntu:20.04'],
};
local platforms_arm = {
develop: ['rockylinux:8'],
'develop-6': ['rockylinux:8'],
};
local any_branch = '**';
local platforms_custom = ['centos:7', 'rockylinux:8', 'debian:10', 'ubuntu:20.04'];
local platforms_arm_custom = ['rockylinux:8'];
local platforms_mtr = ['centos:7', 'rockylinux:8', 'ubuntu:20.04'];
local server_ref_map = {
develop: '10.8',
'develop-6': '10.6-enterprise',
'develop-5': '10.5',
'**': '10.8',
};
local builddir = 'verylongdirnameforverystrangecpackbehavior';
local cmakeflags = '-DCMAKE_BUILD_TYPE=RelWithDebInfo -DPLUGIN_COLUMNSTORE=YES -DPLUGIN_XPAND=NO -DPLUGIN_MROONGA=NO -DPLUGIN_ROCKSDB=NO ' +
'-DPLUGIN_TOKUDB=NO -DPLUGIN_CONNECT=NO -DPLUGIN_SPIDER=NO -DPLUGIN_OQGRAPH=NO -DPLUGIN_SPHINX=NO ' +
'-DWITH_EMBEDDED_SERVER=OFF -DWITH_WSREP=OFF ' +
'-DBUILD_CONFIG=mysql_release -DWITH_UNITTESTS=YES -DCMAKE_GTEST_DISCOVER_TESTS_DISCOVERY_MODE=PRE_TEST';
local gcc_update_alternatives = 'update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 --slave /usr/bin/g++ g++ /usr/bin/g++-10 --slave /usr/bin/gcov gcov /usr/bin/gcov-10 ';
local clang12_update_alternatives = 'update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 100 --slave /usr/bin/clang++ clang++ /usr/bin/clang++-12 && update-alternatives --install /usr/bin/cc cc /usr/bin/clang 100 && update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 100 ';
local rpm_build_deps = 'install -y lz4 systemd-devel git make libaio-devel openssl-devel boost-devel bison snappy-devel flex libcurl-devel libxml2-devel ncurses-devel automake libtool policycoreutils-devel rpm-build lsof iproute pam-devel perl-DBI cracklib-devel expect createrepo ';
local centos7_build_deps = 'yum install -y epel-release centos-release-scl && yum install -y pcre2-devel devtoolset-10 devtoolset-10-gcc cmake3 lz4-devel && ln -s /usr/bin/cmake3 /usr/bin/cmake && . /opt/rh/devtoolset-10/enable ';
local centos8_build_deps = 'dnf install -y gcc-toolset-10 libarchive cmake lz4-devel && . /opt/rh/gcc-toolset-10/enable ';
local rockylinux8_powertools = "dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled powertools ";
local ubuntu18_04_deps = 'apt update && apt install -y gnupg wget && echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-12 main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-12 &&' + clang12_update_alternatives;
local debian10_deps = 'apt update && apt install -y gnupg wget && echo "deb http://apt.llvm.org/buster/ llvm-toolchain-buster-12 main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-12 &&' + clang12_update_alternatives;
local opensuse_build_deps = 'zypper install -y clang12 liblz4-devel cmake libboost_system-devel pcre2-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel gcc-fortran gcc10 gcc10-c++ && ' + clang12_update_alternatives;
local deb_build_deps = 'apt update --yes && apt install --yes --no-install-recommends build-essential devscripts git ccache equivs eatmydata dh-systemd && mk-build-deps debian/control -t "apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends" -r -i ';
local ubuntu20_04_deps = 'apt update --yes && apt install -y g++-10 git && ' + gcc_update_alternatives;
local platformMap(platform, arch) =
local clang_force = if (arch == 'arm64') then ' && export CXX=/usr/bin/clang++ && export CC=/usr/bin/clang ' else '';
local platform_map = {
'opensuse/leap:15': opensuse_build_deps + ' && zypper ' + rpm_build_deps + clang_force + ' && cmake ' + cmakeflags + ' -DRPM=sles15 && make -j$(nproc) package',
'centos:7': centos7_build_deps + ' && yum ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=centos7 && make -j$(nproc) package',
'centos:8': centos8_build_deps + ' && dnf ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=centos8 && make -j$(nproc) package',
'rockylinux:8': rockylinux8_powertools + ' && ' + centos8_build_deps + ' && dnf ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=rockylinux8 && make -j$(nproc) package',
'debian:10': deb_build_deps + " && CMAKEFLAGS='" + cmakeflags + " -DDEB=buster' debian/autobake-deb.sh",
'ubuntu:20.04': ubuntu20_04_deps + ' && ' + deb_build_deps + " && CMAKEFLAGS='" + cmakeflags + " -DDEB=focal' debian/autobake-deb.sh",
};
platform_map[platform];
local testRun(platform) =
local platform_map = {
'opensuse/leap:15': 'ctest -R columnstore: -j $(nproc) --output-on-failure',
'centos:7': 'ctest3 -R columnstore: -j $(nproc) --output-on-failure',
'centos:8': 'ctest3 -R columnstore: -j $(nproc) --output-on-failure',
'rockylinux:8': 'ctest3 -R columnstore: -j $(nproc) --output-on-failure',
'debian:10': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure',
'ubuntu:20.04': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure',
};
platform_map[platform];
local testPreparation(platform) =
local platform_map = {
'opensuse/leap:15': 'zypper install -y git boost-devel libboost_system-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel cppunit-devel snappy-devel cmake',
'centos:7': 'yum -y install epel-release && yum install -y git cppunit-devel cmake3 boost-devel snappy-devel',
'centos:8': 'dnf install -y git lz4 cppunit-devel cmake3 boost-devel snappy-devel',
'rockylinux:8': rockylinux8_powertools + ' && dnf install -y git lz4 cppunit-devel cmake3 boost-devel snappy-devel',
'debian:10': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake',
'ubuntu:18.04': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake g++',
'ubuntu:20.04': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake',
};
platform_map[platform];
local Pipeline(branch, platform, event, arch='amd64') = {
local pkg_format = if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'opensuse/leap' || std.split(platform, ':')[0] == 'rockylinux') then 'rpm' else 'deb',
local init = if (pkg_format == 'rpm') then '/usr/lib/systemd/systemd' else 'systemd',
local mtr_path = if (pkg_format == 'rpm') then '/usr/share/mysql-test' else '/usr/share/mysql/mysql-test',
local socket_path = if (pkg_format == 'rpm') then '/var/lib/mysql/mysql.sock' else '/run/mysqld/mysqld.sock',
local config_path_prefix = if (pkg_format == 'rpm') then '/etc/my.cnf.d/' else '/etc/mysql/mariadb.conf.d/50-',
local img = if (platform == 'centos:7' || std.split(platform, ':')[0] == 'rockylinux') then platform else 'romcheck/' + std.strReplace(platform, '/', '-'),
local regression_ref = if (std.split(branch, '-')[0] == 'develop') then branch else 'develop-6',
local branchp = if (branch == '**') then '' else branch,
local container_tags = if (event == 'cron') then [branch, branch + '-' + std.strReplace(event, '_', '-') + '-${DRONE_BUILD_NUMBER}'] else [branch + '-' + std.strReplace(event, '_', '-') + '-${DRONE_BUILD_NUMBER}'],
local container_version = branch + '/' + event + '/${DRONE_BUILD_NUMBER}/' + arch,
local server_remote = if (std.split(branch, '-')[0] == 'columnstore' || branch == 'develop-6') then 'https://github.com/mariadb-corporation/MariaDBEnterprise' else 'https://github.com/MariaDB/server',
local pipeline = self,
publish(step_prefix='pkg', eventp=event + '/${DRONE_BUILD_NUMBER}'):: {
name: 'publish ' + step_prefix,
image: 'plugins/s3-sync',
when: {
status: ['success', 'failure'],
},
settings: {
bucket: 'cspkg',
access_key: {
from_secret: 'aws_access_key_id',
},
secret_key: {
from_secret: 'aws_secret_access_key',
},
source: 'result',
target: branchp + '/' + eventp + '/' + arch + '/' + std.strReplace(std.strReplace(platform, ':', ''), '/', '-'),
delete: 'true',
},
},
_volumes:: {
mdb: {
name: 'mdb',
path: '/mdb',
},
docker: {
name: 'docker',
path: '/var/run/docker.sock',
},
},
smoke:: {
name: 'smoke',
image: 'docker',
volumes: [pipeline._volumes.docker],
commands: [
'docker run --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name smoke$${DRONE_BUILD_NUMBER} --privileged --detach ' + img + ' ' + init + ' --unit=basic.target',
'docker cp result smoke$${DRONE_BUILD_NUMBER}:/',
if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'rockylinux') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "yum install -y epel-release which rsyslog hostname procps-ng && yum install -y /result/*.' + pkg_format + '"' else '',
if (pkg_format == 'deb') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d',
if (pkg_format == 'deb') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y rsyslog hostname && apt install -y -f /result/*.' + pkg_format + '"' else '',
if (std.split(platform, '/')[0] == 'opensuse') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "zypper install -y which hostname rsyslog && zypper install -y --allow-unsigned-rpm /result/*.' + pkg_format + '"' else '',
// start mariadb and mariadb-columnstore services and run simple query
'docker exec -t smoke$${DRONE_BUILD_NUMBER} systemctl start mariadb',
'docker exec -t smoke$${DRONE_BUILD_NUMBER} systemctl start mariadb-columnstore',
'docker exec -t smoke$${DRONE_BUILD_NUMBER} mariadb -e "create database if not exists test; create table test.t1 (a int) engine=Columnstore; insert into test.t1 values (1); select * from test.t1"',
// restart mariadb and mariadb-columnstore services and run simple query again
'docker exec -t smoke$${DRONE_BUILD_NUMBER} systemctl restart mariadb',
'docker exec -t smoke$${DRONE_BUILD_NUMBER} systemctl restart mariadb-columnstore',
'sleep 10',
'docker exec -t smoke$${DRONE_BUILD_NUMBER} mariadb -e "insert into test.t1 values (2); select * from test.t1"',
],
},
mtr:: {
name: 'mtr',
image: 'docker:git',
[if arch == 'arm64' then 'failure']: 'ignore',
volumes: [pipeline._volumes.docker],
commands: [
'docker run --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --env MYSQL_TEST_DIR=' + mtr_path + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name mtr$${DRONE_BUILD_NUMBER} --privileged --detach ' + img + ' ' + init + ' --unit=basic.target',
'docker cp result mtr$${DRONE_BUILD_NUMBER}:/',
if (std.split(platform, '/')[0] == 'opensuse') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "zypper install -y which hostname rsyslog patch perl-Data-Dumper-Concise perl-Memoize-ExpireLRU && zypper install -y --allow-unsigned-rpm /result/*.' + pkg_format + '"' else '',
if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'rockylinux') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "yum install -y epel-release diffutils which rsyslog hostname patch perl-Data-Dumper perl-Getopt-Long perl-Memoize perl-Time-HiRes cracklib-dicts procps-ng && yum install -y /result/*.' + pkg_format + '"' else '',
if (pkg_format == 'deb') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d',
if (pkg_format == 'deb') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y rsyslog hostname patch && apt install -y -f /result/*.' + pkg_format + '"' else '',
'docker cp mysql-test/columnstore mtr$${DRONE_BUILD_NUMBER}:' + mtr_path + '/suite/',
'docker exec -t mtr$${DRONE_BUILD_NUMBER} chown -R mysql:mysql ' + mtr_path,
// disable systemd 'ProtectSystem' (we need to write to /usr/share/)
'docker exec -t mtr$${DRONE_BUILD_NUMBER} sed -i "/ProtectSystem/d" /usr/lib/systemd/system/mariadb.service',
'docker exec -t mtr$${DRONE_BUILD_NUMBER} systemctl daemon-reload',
'docker exec -t mtr$${DRONE_BUILD_NUMBER} systemctl start mariadb',
'docker exec -t mtr$${DRONE_BUILD_NUMBER} mariadb -e "create database if not exists test;"',
// delay mtr for manual debugging on live instance
'sleep $${MTR_DELAY_SECONDS:-1s}',
'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "cd ' + mtr_path + ' && ./mtr --extern socket=' + socket_path + ' --force --max-test-fail=0 --suite=columnstore/basic,columnstore/bugfixes"',
],
},
mtrlog:: {
name: 'mtrlog',
image: 'docker',
volumes: [pipeline._volumes.docker],
commands: [
'echo "---------- start mariadb service logs ----------"',
'docker exec -t mtr$${DRONE_BUILD_NUMBER} journalctl -u mariadb --no-pager || echo "mariadb service failure"',
'echo "---------- end mariadb service logs ----------"',
'echo',
'echo "---------- start columnstore debug log ----------"',
'docker exec -t mtr$${DRONE_BUILD_NUMBER} cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"',
'echo "---------- end columnstore debug log ----------"',
'echo "---------- end columnstore debug log ----------"',
'docker cp mtr$${DRONE_BUILD_NUMBER}:' + mtr_path + '/var/log /drone/src/result/mtr-logs || echo "missing ' + mtr_path + '/var/log"',
'docker stop mtr$${DRONE_BUILD_NUMBER} && docker rm mtr$${DRONE_BUILD_NUMBER} || echo "cleanup mtr failure"',
],
when: {
status: ['success', 'failure'],
},
},
regression:: {
name: 'regression',
image: 'docker:git',
[if event == 'cron' then 'failure']: 'ignore',
volumes: [pipeline._volumes.docker, pipeline._volumes.mdb],
environment: {
REGRESSION_TESTS: if (event == 'cron') then '' else '${REGRESSION_TESTS:-test000.sh}',
REGRESSION_REF: '${REGRESSION_REF:-' + regression_ref + '}',
REGRESSION_TIMEOUT: {
from_secret: 'regression_timeout',
},
},
commands: [
// clone regression test repo
'git clone --recurse-submodules --branch $$REGRESSION_REF --depth 1 https://github.com/mariadb-corporation/mariadb-columnstore-regression-test',
// where are we now?
'cd mariadb-columnstore-regression-test',
'git rev-parse --abbrev-ref HEAD && git rev-parse HEAD',
'cd ..',
'docker run --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name regression$${DRONE_BUILD_NUMBER} --privileged --detach ' + img + ' ' + init + ' --unit=basic.target',
// copy packages, regresssion test suite and storage manager unit test binary to the instance
'docker cp result regression$${DRONE_BUILD_NUMBER}:/',
'docker cp mariadb-columnstore-regression-test regression$${DRONE_BUILD_NUMBER}:/',
'docker cp /mdb/' + builddir + '/storage/columnstore/columnstore/storage-manager regression$${DRONE_BUILD_NUMBER}:/',
// check storage-manager unit test binary file
'docker exec -t regression$${DRONE_BUILD_NUMBER} ls -l /storage-manager',
if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'rockylinux') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "yum install -y epel-release diffutils tar lz4 wget which rsyslog hostname procps-ng && yum install -y /result/*.' + pkg_format + '"' else '',
if (pkg_format == 'deb') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d',
if (pkg_format == 'deb') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y tar liblz4-tool wget rsyslog hostname && apt install -y -f /result/*.' + pkg_format + '"' else '',
if (std.split(platform, '/')[0] == 'opensuse') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "zypper install -y gzip tar lz4 wget which hostname rsyslog && zypper install -y --allow-unsigned-rpm /result/*.' + pkg_format + '"' else '',
// copy test data for regression test suite
'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "wget -qO- https://cspkg.s3.amazonaws.com/testData.tar.lz4 | lz4 -dc - | tar xf - -C mariadb-columnstore-regression-test/"',
// set mariadb lower_case_table_names=1 config option
'docker exec -t regression$${DRONE_BUILD_NUMBER} sed -i "/^.mariadb.$/a lower_case_table_names=1" ' + config_path_prefix + 'server.cnf',
// set default client character set to utf-8
'docker exec -t regression$${DRONE_BUILD_NUMBER} sed -i "/^.client.$/a default-character-set=utf8" ' + config_path_prefix + 'client.cnf',
// start mariadb and mariadb-columnstore services
'docker exec -t regression$${DRONE_BUILD_NUMBER} systemctl start mariadb',
'docker exec -t regression$${DRONE_BUILD_NUMBER} systemctl start mariadb-columnstore',
// delay regression for manual debugging on live instance
'sleep $${REGRESSION_DELAY_SECONDS:-1s}',
// run regression test000 on pull request and manual (may be overwritten by env variable parameter) build events. on other events run all tests
'docker exec -t --workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest regression$${DRONE_BUILD_NUMBER} timeout -k 1m -s SIGKILL --preserve-status $${REGRESSION_TIMEOUT:-10h} ./go.sh --sm_unit_test_dir=/storage-manager --tests=$${REGRESSION_TESTS}',
],
},
smokelog:: {
name: 'smokelog',
image: 'docker',
volumes: [pipeline._volumes.docker],
commands: [
'echo "---------- start mariadb service logs ----------"',
'docker exec -t smoke$${DRONE_BUILD_NUMBER} journalctl -u mariadb --no-pager || echo "mariadb service failure"',
'echo "---------- end mariadb service logs ----------"',
'echo',
'echo "---------- start columnstore debug log ----------"',
'docker exec -t smoke$${DRONE_BUILD_NUMBER} cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"',
'echo "---------- end columnstore debug log ----------"',
'docker stop smoke$${DRONE_BUILD_NUMBER} && docker rm smoke$${DRONE_BUILD_NUMBER} || echo "cleanup smoke failure"',
],
when: {
status: ['success', 'failure'],
},
},
regressionlog:: {
name: 'regressionlog',
image: 'docker',
volumes: [pipeline._volumes.docker],
commands: [
'echo "---------- start columnstore regression short report ----------"',
'docker exec -t --workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest regression$${DRONE_BUILD_NUMBER} cat go.log || echo "missing go.log"',
'echo "---------- end columnstore regression short report ----------"',
'echo',
'docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs.tgz /drone/src/result/ || echo "missing testErrorLogs.tgz"',
'docker exec -t --workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest regression$${DRONE_BUILD_NUMBER} bash -c "tar czf testErrorLogs2.tgz *.log /var/log/mariadb/columnstore" || echo "failed to grab regression results"',
'docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs2.tgz /drone/src/result/ || echo "missing testErrorLogs.tgz"',
'docker stop regression$${DRONE_BUILD_NUMBER} && docker rm regression$${DRONE_BUILD_NUMBER} || echo "cleanup regression failure"',
],
when: {
status: ['success', 'failure'],
},
},
dockerfile:: {
name: 'dockerfile',
image: 'alpine/git',
commands: [
'git clone --depth 1 https://github.com/mariadb-corporation/mariadb-enterprise-columnstore-docker',
"sed -i 's|dlm.mariadb.com/enterprise-release-helpers/mariadb_es_repo_setup|cspkg.s3.amazonaws.com/cs_repo|' mariadb-enterprise-columnstore-docker/Dockerfile",
],
},
dockerhub:: {
name: 'dockerhub',
image: 'plugins/docker',
environment: {
VERSION: container_version,
},
settings: {
repo: 'mariadb/enterprise-columnstore-dev',
context: 'mariadb-enterprise-columnstore-docker',
dockerfile: 'mariadb-enterprise-columnstore-docker/Dockerfile',
build_args_from_env: ['VERSION'],
tags: container_tags,
username: {
from_secret: 'dockerhub_user',
},
password: {
from_secret: 'dockerhub_password',
},
},
},
kind: 'pipeline',
type: 'docker',
name: std.join(' ', [branch, platform, event, arch]),
platform: { arch: arch },
// [if arch == 'arm64' then 'node']: { arch: 'arm64' },
clone: { depth: 10 },
steps: [
{
name: 'submodules',
image: 'alpine/git',
commands: [
'git submodule update --init --recursive',
'git config cmake.update-submodules no',
'git rev-parse --abbrev-ref HEAD && git rev-parse HEAD',
],
},
{
name: 'clone-mdb',
image: 'alpine/git',
volumes: [pipeline._volumes.mdb],
environment: {
SERVER_REF: '${SERVER_REF:-' + server_ref_map[branch] + '}',
SERVER_REMOTE: '${SERVER_REMOTE:-' + server_remote + '}',
SERVER_SHA: '${SERVER_SHA:-' + server_ref_map[branch] + '}',
},
commands: [
'echo $$SERVER_REF',
'echo $$SERVER_REMOTE',
'mkdir -p /mdb/' + builddir + ' && cd /mdb/' + builddir,
'git config --global url."https://github.com/".insteadOf [email protected]:',
'git -c submodule."storage/rocksdb/rocksdb".update=none -c submodule."wsrep-lib".update=none -c submodule."storage/columnstore/columnstore".update=none clone --recurse-submodules --depth 200 --branch $$SERVER_REF $$SERVER_REMOTE .',
'git reset --hard $$SERVER_SHA',
'git rev-parse --abbrev-ref HEAD && git rev-parse HEAD',
'git config cmake.update-submodules no',
'rm -rf storage/columnstore/columnstore',
'cp -r /drone/src /mdb/' + builddir + '/storage/columnstore/columnstore',
],
},
{
name: 'build',
image: img,
volumes: [pipeline._volumes.mdb],
environment: {
DEBIAN_FRONTEND: 'noninteractive',
BUILDPACKAGE_FLAGS: '-b', // Save time and produce only binary packages, not source
},
commands: [
'cd /mdb/' + builddir,
// Remove Debian build flags that could prevent ColumnStore from building
"sed '/-DPLUGIN_COLUMNSTORE=NO/d' -i debian/rules",
// Tweak debian packaging stuff
"sed -i -e '/Package: mariadb-backup/,/^$/d' debian/control",
"sed -i -e '/Package: mariadb-plugin-connect/,/^$/d' debian/control",
"sed -i -e '/Package: mariadb-plugin-gssapi*/,/^$/d' debian/control",
"sed -i -e '/Package: mariadb-plugin-xpand*/,/^$/d' debian/control",
'sed "/Package: mariadb-plugin-mroonga/,/^$/d" -i debian/control',
'sed "/Package: mariadb-plugin-rocksdb/,/^$/d" -i debian/control',
'sed "/Package: mariadb-plugin-spider/,/^$/d" -i debian/control',
'sed "/Package: mariadb-plugin-oqgraph/,/^$/d" -i debian/control',
'sed "/ha_sphinx.so/d" -i debian/mariadb-server-*.install',
'sed "/Package: libmariadbd19/,/^$/d" -i debian/control',
'sed "/Package: libmariadbd-dev/,/^$/d" -i debian/control',
// Disable Galera
"sed -i -e 's/Depends: galera.*/Depends:/' debian/control",
"sed -i -e 's/\"galera-enterprise-4\"//' cmake/cpack_rpm.cmake",
"sed -i -e '/wsrep/d' debian/mariadb-server-*.install",
// Leave test package for mtr
"sed -i '/(mariadb|mysql)-test/d;/-test/d' debian/autobake-deb.sh",
"sed -i '/test-embedded/d' debian/mariadb-test.install",
// Deb dependencies from server scripts
if (pkg_format == 'deb') then "apt-cache madison liburing-dev | grep liburing-dev || sed 's/liburing-dev/libaio-dev/g' -i debian/control && sed '/-DIGNORE_AIO_CHECK=YES/d' -i debian/rules && sed '/-DWITH_URING=yes/d' -i debian/rules && apt-cache madison libpmem-dev | grep 'libpmem-dev' || sed '/libpmem-dev/d' -i debian/control && sed '/-DWITH_PMEM/d' -i debian/rules && sed '/libfmt-dev/d' -i debian/control" else '',
// Change plugin_maturity level
// "sed -i 's/BETA/GAMMA/' storage/columnstore/CMakeLists.txt",
testPreparation(platform),
platformMap(platform, arch),
if (pkg_format == 'rpm') then 'createrepo .' else 'dpkg-scanpackages ../ | gzip > ../Packages.gz',
],
},
{
name: 'unittests',
image: img,
volumes: [pipeline._volumes.mdb],
environment: {
DEBIAN_FRONTEND: 'noninteractive',
},
commands: [
'cd /mdb/' + builddir,
testPreparation(platform),
testRun(platform),
],
},
{
name: 'list pkgs',
image: 'docker:git',
volumes: [pipeline._volumes.mdb],
commands: [
'cd /mdb/' + builddir,
'mkdir -p /drone/src/result',
'echo "engine: $DRONE_COMMIT" > buildinfo.txt',
'echo "server: $$(git rev-parse HEAD)" >> buildinfo.txt',
'echo "buildNo: $DRONE_BUILD_NUMBER" >> buildinfo.txt',
'cp -r ' + (if (pkg_format == 'deb') then '../Packages.gz ../' else 'repodata ') + '*.' + pkg_format + ' buildinfo.txt /drone/src/result/',
'ls -l /drone/src/result',
'echo "check columnstore package:"',
'ls -l /drone/src/result | grep columnstore',
],
},
] +
[pipeline.publish()] +
(if (event == 'cron') || (event == 'push') then [pipeline.publish('pkg latest', 'latest')] else []) +
(if (event != 'custom') && (platform == 'rockylinux:8') && (arch == 'amd64') && (branch != 'develop-5') then [pipeline.dockerfile] + [pipeline.dockerhub] else []) +
[pipeline.smoke] +
[pipeline.smokelog] +
(if (std.member(platforms_mtr, platform)) then [pipeline.mtr] + [pipeline.mtrlog] + [pipeline.publish('mtr')] else []) +
(if (event == 'cron' && std.member(platforms_mtr, platform)) || (event == 'push') then [pipeline.publish('mtr latest', 'latest')] else []) +
[pipeline.regression] +
[pipeline.regressionlog] +
[pipeline.publish('regression')] +
(if (event == 'cron') || (event == 'push') then [pipeline.publish('regression latest', 'latest')] else []),
volumes: [pipeline._volumes.mdb { temp: {} }, pipeline._volumes.docker { host: { path: '/var/run/docker.sock' } }],
trigger: {
event: [event],
branch: [branch],
} + (if event == 'cron' then {
cron: ['nightly-' + std.strReplace(branch, '.', '-')],
} else {}),
};
local FinalPipeline(branch, event, arch='amd64') = {
kind: 'pipeline',
name: std.join(' ', ['after', branch, event]),
steps: [
{
name: 'notify',
image: 'plugins/slack',
settings: {
webhook: {
from_secret: 'slack_webhook',
},
template: '*' + event + (if event == 'pull_request' then ' <https://github.com/{{repo.owner}}/{{repo.name}}/pull/{{build.pull}}|#{{build.pull}}>' else '') +
' build <{{build.link}}|{{build.number}}> {{#success build.status}}succeeded{{else}}failed{{/success}}*.\n\n*Branch*: <https://github.com/{{repo.owner}}/{{repo.name}}/tree/{{build.branch}}|{{build.branch}}>\n*Commit*: <https://github.com/{{repo.owner}}/{{repo.name}}/commit/{{build.commit}}|{{truncate build.commit 8}}> {{truncate build.message.title 100 }}\n*Author*: {{ build.author }}\n*Duration*: {{since build.started}}\n*Artifacts*: https://cspkg.s3.amazonaws.com/index.html?prefix={{build.branch}}/{{build.event}}/{{build.number}}',
},
},
],
trigger: {
event: [event],
branch: [branch],
status: [
'success',
'failure',
],
} + (if event == 'cron' then { cron: ['nightly-' + std.strReplace(branch, '.', '-')] } else {}),
depends_on: std.map(function(p) std.join(' ', [branch, p, event, arch]), platforms[branch]) +
(if branch == 'develop' then std.map(function(p) std.join(' ', [branch, p, event, 'arm64']), platforms_arm[branch]) else []),
};
[
Pipeline(b, p, e)
for b in std.objectFields(platforms)
for p in platforms[b]
for e in events
] +
[
Pipeline(b, p, e, 'arm64')
for b in std.objectFields(platforms_arm)
for p in platforms_arm[b]
for e in events
] +
[
FinalPipeline(b, e)
for b in std.objectFields(platforms)
for e in events
]
+
[
Pipeline(any_branch, p, 'custom')
for p in platforms_custom
] +
[
Pipeline(any_branch, p, 'custom', 'arm64')
for p in platforms_arm_custom
]