-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathcheckQuality
executable file
·2410 lines (1986 loc) · 96.5 KB
/
checkQuality
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/local/bin/bash
#########################################################################################
# Linux Shell script for running SonarQube analyzer, Jan. 2016
# Steps:
# 1. set common values (C++ and python)
# 2. run tools for C++ code analysis
# 3. run tools for python code analysis
# 4. write the sonar-project.properties file and run sonar-runner
#
# SVN repository location : $URL: http://euclid.esac.esa.int/svn/EC/SGS/ST/4-2-09-TOOLS/CODEEN/CWGs/Quality/QualCheck/tags/1.4.1/checkQuality.sh $:
# Revision of last commit : $Rev: 15428 $:
# Author of last commit : $Author: dbagot $:
# Date of last commit : $Date: 2016-10-03 15:50:12 +0200 (Mon, 03 Oct 2016) $:
#########################################################################################
# Comments for maintenance...
# shell style guide: https://google-styleguide.googlecode.com/svn/trunk/shell.xml
# Bash min version: 4.0
# Variables:
# - most of the variables are global (they are declared local or read-only when possible)
# - several global variables are initialized in function names get_<globalvar>
# - bash 4 allows associative arrays (used here)
# Files modified by this script:
# - ./reports directory with txt and xml files
# - ./sonar-project.properties created/modified
# - *.gcda and *.gcno produced by coverage binaries in sub-folders
# - envr var PYTHONPATH and VERA_ROOT possibly modified
# ----------------------------------------------------------------------------------------
# Script variables
# ----------------------------------------------------------------------------------------
scriptFullName=$0
scriptName=$(basename $0)
lineSep="--------------------------------------------------------------------------------"
# debug output (short vars to facilitate the display of debug messages)
F=FUNCNAME
L=LINENO
# ----------------------------------------------------------------------------------------
# default arguments
# ----------------------------------------------------------------------------------------
languages="c++ python" # selected languages for code analysis
targetMake="none" # no forced generation (expected to be done before)
logLevel="info" # information displayed (no debug mode, no quiet mode)
staticTools="gcc cppcheck vera++" # list of possible quality tools for static analysis (rats not included TBC)
dynamicTools="valgrind gcovr coverage" # list of possible quality tools for dynamic analysis
#dynamicTools="valgrind ctest" # list of possible quality tools for dynamic analysis
sonar="yes" # activate sonar by default (with default quality gate)
# ----------------------------------------------------------------------------------------
# Quality analysis variables (for SonarQube)
# ----------------------------------------------------------------------------------------
# Sonar http local adress
sonarQubeURL_LODEEN="http://localhost:9000/sonar"
sonarQubeURL_CODEEN="https://apceuclidapp.in2p3.fr/sonar"
# default= LODEEN (SonarQube filled in getSonarQubeInfos
declare -g -A SonarQube
SonarQube[URL]=$sonarQubeURL_LODEEN
# mandatory with ctest
ctestToJunitFileName="$HOME/ctest-to-junit.xsl"
# folder containing the reports from analysis
#reportsDir=${scriptName}_reports
reportsDir="PAQA"
reportsDirTMP=".PAQA.tmp"
reportsDirCpp="${reportsDir}/CPP"
reportsDirPy="${reportsDir}/PYTHON"
logbook=${reportsDir}/${scriptName}_report.txt
runToolsScript=${reportsDir}/runQualityTools.sh
sonarQube_reportFile=""
cppCoverage_reportHTMLFile="${reportsDirCpp}/cpp-coverage-report.html"
pythonCoverage_reportHTMLDir="${reportsDirPy}/py-coverage-HTMLreports"
# below: const necessary for grep operations on several reports produced by valgrind and unit tests
valgrindReport="cpp-valgrind-report"
#xunitReport="cpp-xunit-report"
xunitReport="cpp-xunit-report"
py_xunitReport="py-unitests-report"
userTestReport="cpp-xunit-report-user"
userCppRunReport="cpp-user-report"
userPyRunReport="py-user-report"
#veraReport="cpp-vera-report"
# gcc warnings
sonar_gcc_reportFile="gcc-warnings.log" # redefined below in main (depends on BINARY_TAG)
# paths to the (XML sonar format) reports
sonar_cxx_cppcheck_reportFile="${reportsDirCpp}/cpp-cppcheck-report.xml"
sonar_cxx_vera_reportFile="${reportsDirCpp}/cpp-vera-report.xml"
sonar_cxx_rats_reportFile="${reportsDirCpp}/cpp-rats-report.xml"
sonar_cxx_coverage_reportFile="${reportsDirCpp}/cpp-coverage-report.xml"
sonar_cxx_xunit_reportFile=""
sonar_cxx_valgrind_reportFile="${reportsDirCpp}/${valgrindReport}*.xml"
sonar_python_coverage_reportFile="${reportsDirPy}/py-coverage-report.xml"
sonar_python_xunit_reportFile=""
# declare global associative arrays
# keys: tools as gcc,..Values=version of the tool. Filled in get_toolVersion
declare -g -A toolVersion
# keys: info of the project: name,... Filled in get_project
declare -g -A project
# keys: elements lib ("USE" in CMaleLists.txt). Values=lib version. Filled in get_project
declare -g -A projectCodePaths
# keys: name src include python tests_python Filled in get_projectCodePaths
declare -g -A projectUSE
# list of elements include/import paths.Keys: "c++" and "python". Filled in get_elementsLib
declare -g -A elementsLib
# list of system include/import paths.Keys: "c++" and "python". Filled in get_systemLib
declare -g -A systemLib
# list of all necessary tools and var to execute complete analysis
declare -g -A requiredEnv
# list of all the binary C++ and python founded (keys : o2g and cov)
declare -g -A cppTestName
declare -g -A cppTestPath
declare -g -A cppTestArgs
declare -g -A cppTestType
declare -g -A pythonTestName
declare -g -A pythonTestPath
declare -g -A pythonTestArgs
declare -g -A pythonTestType
# list of all the executables added in options C++ and python founded (key : incremental integer)
declare -g -A cppUserTestsWithOptions
declare -g -A pyUserTestsWithOptions
# part dedicated to SonarQube webservices
declare -g -A projectNamebyId
declare -g -A projectIdbyKey
declare -g -A projectKeysbyId
declare -g -A projectKeysbyName
declare -g -A projectQualityGatebyKeyQG
declare -g -A projectQualityGatebyKey
# ----------------------------------------------------------------------------------------
# Functions
# ----------------------------------------------------------------------------------------
#########################################################
# display usage
# In Globals: scriptName languages staticTools
# dynamicTools sonar $targetMake logLevel
#########################################################
function usage
{
echo "
$scriptName
[-l|--log-level=<none|info|debug>]
[-L|--languages=<languages>]
[-s|--static-tools=<tools>]
[-d|--dynamic-tools=<tools>]
[-sonar=<no>|<yes>|<quality gate>]
[-m|--make=<none|configure|all|clean|purge|test|install>]
[-bincpp|--binary-cpp=<C++ executable with options>]
[-binpy|--binary-py=<Python program with options>]
[-p|--projects [codeen]
[-r|--report=[project key] [diff] [codeen]
[-e|--envt]
[-u] [-h]
[-v] --version
[list of modules]
default mode:
$scriptName --languages \"$languages\" --static-tools \"$staticTools\" --dynamic-tools \"$dynamicTools\" -sonar $sonar
--make $targetMake --log-level $logLevel"
} # usage
##############################################################
# display help
# In Globals: scriptName SonarQube languages staticTools
# dynamicTools sonar $targetMake logLevel
##############################################################
function help
{
echo "
NAME
$scriptName - - Tool for static and dynamic C++ and Python code analysis (sonarQube framework)
SYNOPSIS
$scriptName is a command-line tool that tries to detect bugs that your C/C++ compiler or Python interpreter doesn't see.
It launches several quality rools (such as cppcheck or pylint), gather their results into XML files and run finally the local sonar-runner executable
Results from this local analysis is displayed in your local web browser, on $sonarQubeURL_LODEEN
"
usage
echo "
OPTIONS
[-L|--languages=<languages>]
select only source written in a given language (default value is \"$languages\")
example: '> $scriptName --languages python' : analyze only python source code
[-s|--static-tools=<names of quality tools>]
select tools to be launched for the static analysis. Default list is \"$staticTools\"
static analysis: means that the selected source code will be analyzed without any expected generation aor execution
Note for gcc: \"$scriptName\" reads the \"make\" output text file \"$sonar_gcc_reportFile\" under \"build.\$BINARY_TAG\"
examples:
'> $scriptName --static-tools \"$staticTools\"' : static analysis only (no dynamic analysis) (\"$languages\")
'> $scriptName -s \"cppcheck vera++\"' : static analysis for C++ code only with cppcheck and vera++ tools
[-d|--dynamic-tools=<names of quality tools>]
select tools to be launched for the dynamic analysis. Default list is \"$dynamicTools\"
dynamic analysis: means that a generation (see -m|--make option) and unit tests execution are mandatory
the structural coverage of the source code by unit tests will be thereafter measured and a memory analysis (for C++ code) done
examples:
'> $scriptName --dynamic-tools \"$dynamicTools\"' : dynamic analysis only
'> $scriptName -d coverage' : dynamic analysis with coverage.py only
est
[-sonar=<yes>|<no>|<quality gate>]
yes: Results from this local analysis is displayed in your local web browser, on $sonarQubeURL_LODEEN
Quality gate is set in the sonar server configuration
no: sonar is not launched
quality gate: same as '-sonar yes'but specifying the quality gate (eg PROTOTYPE, DEVELOPMENT, PRODUCTION)
The list of the available quality gate on your local sonar is displayed in '${SonarQube[URL]}/quality_gates'
examples:
'> $scriptName -sonar no' : deactivate the default option (so no sonar performance)
'> $scriptName -sonar PROTOTYPE' : run sonar with quality gate set to 'PROTOTYPE'
[-m|--make=<none|configure|all|clean|purge|test|install>]
do a generation before the analysis, i.e. run make <target> before launching dynamic tools as gcovr, valgrind or coverage.py
Important note: this generation will perfomed with the two build options (o2g and cov)
examples:
'> $scriptName --make all' : do a \"make all\" before running all the tools
'> $scriptName - m \"purge all\"' : clean the (generation) environment and do a generation
[-bincpp|--binary-cpp=<C++ executable with options>]
[-binpy|--binary-python=<Python program with options>]
run a executable (with options) in the PATH or generated under \"./build/\$BINARY_TAG\" relative folder (e.g. \"./build.$BINARY_TAG_o2g\")
examples:
'> $scriptName -L \"python\" -binpy \"scripts/pyfirststep --workdir=/home/user --simvisproduct=Output.xml --skyposition=output.txt\"'
'> $scriptName -L \"c++\" -s \"\" -d \"gcovr\" -bincpp \"py.test ./ThisModule/tests/python\"
[-p|--projects [codeen] : list of the current projects in sonarQube (in LODEEN by default, in CODEEN is optional argument \"codeen\" added)
the table shows the projects (i.e. generic names) as \"SIM_TIPS\" and all their versions analyzed in sonarQube, as \"SIM_TIPS-1.2.3\"
examples:
'> $scriptName --projects' : list of the projects and project keys (i.e. versions) which analysis exist in LODEEN platform
'> $scriptName -p codeen' : list of the projects and projects keys in CODEEN
[-r|--report [project key] [codeen] : summary of the sonarQube analysis (in LODEEN by default, in CODEEN is optional argument \"codeen\" added)
project key: given par the option \"--projects [codeen]\" (see below). If argument done, the project is from the local directory.
if \"diff\" added as option, a colorized diff tool is run (vimdiff: type \":qa\" to quit)
examples:
'> $scriptName --report PHZ_Alexandria-TRUNK' : summary of the analysis of the project key PHZ_Alexandria (trunk) in LODEEN platform
'> $scriptName -r SIM_TIPS-1.2.3 codeen' : summary of the analysis of the project SIM_TIPS (version 1.2.3) existing in LODEEN platform
'> $scriptName -r diff' : comparison between the current and the previous analysis using vimdiff (LODEEN platform)
[-e|--envt] [\"tools\"]
print all the versions of the required tools for a complete analysis and also print project information on current directory
Note: the previous analysis directory \"$reportsDir\" is removed. Last check step is not done if optional agument is set to \"tools\"
example: '> $scriptName --envt', '> $scriptName -e tools'
[-l|--log-level=<none|info|debug>]
print information with a givel log level
example: '> $scriptName --log-level debug' : debug mode (verbose mode !)
[-u] [-h]
usage and (this) help
[-v] --version
version (SVN revision) of $scriptName
[list of modules] run static or/and dynamic analysis on selected (elements) modules
examples:
'> $scriptName GridContainer MathUtils'
'> $scriptName GridContainer -s \"gcc cppcheck\" -sonar no'
COPYRIGHT
Copyright (C) 2012-2020 Euclid Science Ground Segment
This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General
Public License as published by the Free Software Foundation; either version 3.0 of the License, or (at your option)
any later version.
This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
details.
You should have received a copy of the GNU Lesser General Public License along with this library; if not, write to
the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
"
} # help
#######################################
# display messages for information
# In Globals: logLevel logboog
# Arguments: message
#######################################
function info
{
if [ "$logLevel" == "info" -o "$logLevel" == "debug" ] ; then
#echo "[$scriptName] : $@"| tee -a $logbook
echo -e "\e[92m$@\e[0m" | tee -a $logbook
#echo "$@" | tee -a $logbook
fi
}
#######################################
# display messages for debug
# In Globals: logLevel logbook
# Arguments: title + message
#######################################
function debug
{
local title=$1
shift
if [ "$logLevel" == "debug" ] ; then
echo "[DEBUG $title]: $@" | tee -a $logbook
else
echo "[DEBUG $title]: $@" >> $logbook
fi
}
#######################################
# display error messages on &2 and exit (code = 1)
# In Globals: scriptName logLevel logbook
# Arguments: title + message
#######################################
function error
{
local title=$1
shift
#echo "[ERROR $scriptName line $title]: $@" | tee -a $logbook
echo -e "\e[91m[ERROR $scriptName line $title]: $@\e[0m" | tee -a $logbook
exit 1
}
#######################################
# display warning messages on &2
# In Globals: scriptName logLevel logbook
# Arguments: title + message
#######################################
function warn
{
local title=$1
shift
#echo "[WARN $scriptName line $title]: $@" | tee -a $logbook
echo -e "\e[91m[WARN $scriptName line $title]: $@\e[0m" | tee -a $logbook
}
#######################################
# display addscript lines
# In Globals: runToolsScript
# Arguments: addscript line
#######################################
function addscript
{
#echo "[WARN $scriptName line $title]: $@" | tee -a $logbook
echo -e "$@" >> $runToolsScript
}
#######################################
# existsAndNotEmpty
# Arguments: directory or file
# return TRUE (0) if file/directory exists and is not empty else returns FALSE (1)
#######################################
function existsAndNotEmpty
{
local item=$1
debug ${!F}_${!L} "----------------> BEGIN: existsAndNotEmpty $item"
local returnCode=1 # FALSE
if [ -f "$item" ] ; then
[ -s "$item" ] && returnCode=0 # TRUE: file exists and not empty
elif [ -d "$item" ] ; then
[ "$(ls -A $item)" ] && returnCode=0 # TRUE: directory exists and contains file(s)
fi
debug ${!F}_${!L} "----------------> END: existsAndNotEmpty with return code=$returnCode"
return $returnCode
} # existsAndNotEmpty
#######################################
# initialize addscript lines script
# In Globals: runToolsScript
#######################################
function initRunToolsScript
{
debug ${!F}_${!L} "touch $runToolsScript"
touch $runToolsScript
chmod a+x $runToolsScript
addscript "#!/bin/bash"
}
#######################################
# initialize logbook
# In Globals: logbook reportsDir reportsDirTMP
# Arguments: "tmp" or "append" (optional $1)
#######################################
function initLogbook
{
local withTmpOption=$1
if [[ "$withTmpOption" == "tmp" ]] ; then
# shell run with only "display" commands (no persistent logbook)
reportsDir=$reportsDirTMP
logbook=${reportsDir}/${scriptName}_report.txt
mkdir ${reportsDir} 2>/dev/null
# initialize the logbook for next messages (info, debug, warn, error)
touch $logbook
else
if [[ "$withTmpOption" == "append" ]] && [[ -d $reportsDir ]] ; then
# echo "The folder ${reportsDir} exists and will be used by the command with \"$withTmpOption\""
debug ${!F}_${!L} "The folder ${reportsDir} exists and will be used by the command with \"$withTmpOption\""
else
# persistent logbook: save the previous before launching commands
# echo "${reportsDir} deleted"
rm -rf ${reportsDir}/${reportsDir}_previous 2>/dev/null
#read -p "rm -rf ${reportsDir}/${reportsDir}_previous"
mv ${reportsDir} ${reportsDir}_previous 2>/dev/null
#read -p "mv ${reportsDir} ${reportsDir}_previous"
mkdir ${reportsDir} 2>/dev/null
#read -p "mkdir ${reportsDir}"
#TODO mkdir C++ et Py sub-folders only when necessary
mkdir ${reportsDirCpp} 2>/dev/null
mkdir ${reportsDirPy} 2>/dev/null
#read -p "mkdir ${reportsDirCpp} ${reportsDirPy}"
mv ${reportsDir}_previous ${reportsDir} 2>/dev/null
#read -p "mv ${reportsDir}_previous ${reportsDir} "
mv sonar-project.properties ${reportsDir}/${reportsDir}_previous 2>/dev/null
# initialize the logbook for next messages (info, debug, warn, error)
touch $logbook
#read -p "touch $logbook"
debug ${!F}_${!L} "The previous folder ${reportsDir} has been moved in ${reportsDir}/${reportsDir}_previous"
fi
fi
} # initLogbook
#######################################
# select modules
# In Globals:
# Arguments: list of modules or nothing
#######################################
function selectModules
{
local myModulesInput=$@
local myModules=""
local module=""
local myModOK=""
local key=""
local dir=""
local values=""
local term=""
debug ${!F}_${!L} "----------------> BEGIN: selectModules $myModulesInput"
for module in $myModulesInput
do
debug ${!F}_${!L} "module=${module}."
[[ "$module" == "." ]] && break
# supress / and ./
myModOK=$(echo $module | sed "s,\.\/,,g" | sed "s,\/,,g" )
[[ ! -d $myModOK ]] && error $LINENO "module \"$myModOK\" does not exist"
myModules="$myModules $myModOK"
done
debug ${!F}_${!L} "Selection of modules = $myModules."
if [[ "$myModules" ]] ; then
for key in modules src include tests_src python tests_python ; do
values=""
for dir in ${projectCodePaths[$key]} ; do
for module in $myModules ; do
term=""
if [[ "$key" == "modules" ]] ; then
term=$(echo $dir | grep "^${module}$" )
else
term=$(echo $dir | grep "^${module}\/")
fi
[[ "$term" ]] && values="$term $values"
done # for module
done # for dir
projectCodePaths[$key]=$values
debug ${!F}_${!L} "Selected targets: projectCodePaths[$key]=${projectCodePaths[${key}]}"
done # for key
fi # if [[ "$myModules"
debug ${!F}_${!L} "----------------> END: selectModules"
} # selectModules
#######################################
# exit script (run with "display" commands, i.e. no persistent logbook)
# In Globals: reportsDirTMP
#######################################
function exitScript
{
rm -rf ${reportsDir} 2>/dev/null
exit 1
} # exitScript
##################################################################
# display this script version
# In Globals: scriptFullName
##################################################################
function printVersion
{
version=$( grep "^# SVN repository location" $scriptFullName | egrep -o '(tags|branches)/[^/]+|trunk' | egrep -o '[^/]+$')
revision=$( grep "^# Revision of last commit" $scriptFullName | awk '{print $8}')
echo "$scriptName version $version (svn revision $revision)"
} # printVersion
###################################################################################################
# getSonarQubeQualityGates
# In Globals: sonarQubeURL projectKeybyId
# Out Globals: sonarQube projectQualityGatebyKeyQG projectQualityGatebyKey
###################################################################################################
function getSonarQubeQualityGates
{
debug ${!F}_${!L} "----------------> BEGIN: getSonarQubeQualityGates"
local tmpFile="/tmp/getSonarQubeQualityGates$$.txt"
local tmpFile2="/tmp/getSonarQubeQualityGates2$$.txt"
debug ${!F}_${!L} "getSonarQubeQualityGates sonarQube[URL] = ${SonarQube[URL]}"
runCurl "${SonarQube[URL]}/api/qualitygates/list" $tmpFile
cat $tmpFile | jq -jr '.qualitygates[] | "\(.id) \(.name)\n"' > $tmpFile2 2>/dev/null
# output: "8 DEVELOPMENT\n9 PRODUCTION\n7 PROTOTYPE\n"
while read line
do
QGid=$(echo $line | awk '{print $1}')
QGname=$(echo $line | awk '{for (i=2; i<=NF; i++) printf $i " "; }')
debug ${!F}_${!L} "QGid = $QGid, QGname=$QGname"
projectQualityGatebyKeyQG[$QGid]=$QGname
debug ${!F}_${!L} "projectQualityGatebyKeyQG[$QGid]=${projectQualityGatebyKeyQG[$QGid]}"
done < $tmpFile2
rm $tmpFile2 2>/dev/null
# set the quality gate "default" for all the projects
for id in ${!projectKeybyId[@]}; do
projectQualityGatebyKey[$id]="default"
done
# add all the projects by Quality Gate
for key in ${!projectQualityGatebyKeyQG[@]}; do
name=${projectQualityGatebyKeyQG[${key}]}
debug ${!F}_${!L} "id QG=$key, quality gate=${name}."
runCurl "${SonarQube[URL]}/api/qualitygates/search?gateId=$key&selected=all" $tmpFile
cat $tmpFile | jq -r '.results[] | "\(.id) \(.selected) "' > $tmpFile2 2>/dev/null
# "11529 false \n10920 true \n7118 false"
while read line
do
selected=$(echo $line | awk '{print $2}')
id=$(echo $line | awk '{print $1}')
#debug ${!F}_${!L} "id = $id, selected=$selected"
[[ "$selected" == "true" ]] && projectQualityGatebyKey[$id]=$name
#debug ${!F}_${!L} "projectQualityGatebyKey[$id]=${projectQualityGatebyKey[$id]}"
done < $tmpFile2
rm $tmpFile2 2>/dev/null
done
rm $tmpFile 2>/dev/null
debug ${!F}_${!L} "----------------> END: getSonarQubeQualityGates"
} #getSonarQubeQualityGates
###################################################################################################
# getSonarQubeProjects
# In Globals: SonarQube
# Out Globals: projectNamebyId, projectKeybyId, projectKeysbyName, projectIdbyKey
###################################################################################################
function getSonarQubeProjects
{
debug ${!F}_${!L} "----------------> BEGIN: getSonarQubeProjects"
local tmpFile="/tmp/getSonarQubeProjects$$.txt"
local tmpFile2="/tmp/getSonarQubeProjects2$$.txt"
runCurl "${SonarQube[URL]}/api/projects/index" $tmpFile
cat $tmpFile | jq -r '.[] | "\(.k) \(.id) \(.nm)"' > $tmpFile2 2>/dev/null
# output: "CT_DEVWS2-1.0 11610 CT_DEVWS2\nCT_Elements:2.2 5698 CT_Elements\n..."
while read line
do
name=$(echo $line | awk '{for (i=3; i<NF; i++) printf $i " "; print $NF}')
key=$(echo $line | awk '{print $1}')
id=$(echo $line | awk '{print $2}')
#read key name <<< $line
#debug ${!F}_${!L} "key=$key, name=$name, id=$id"
projectNamebyId[$id]=$name
projectIdbyKey[${key}]=$id
projectKeybyId[$id]=$key
debug ${!F}_${!L} "projectNamebyId[$id]=${projectNamebyId[$id]}."
debug ${!F}_${!L} "projectIdbyKey[$key]=${projectIdbyKey[$key]}."
debug ${!F}_${!L} "projectKeybyId[$id]=${projectKeybyId[$id]}."
done < $tmpFile2
rm $tmpFile 2>/dev/null
rm $tmpFile2 2>/dev/null
for id in ${!projectKeybyId[@]}; do
key=${projectKeybyId[$id]}
name=${projectNamebyId[$id]}
projectKeysbyName[${name}]="$key ${projectKeysbyName[${name}]}"
debug ${!F}_${!L} "projectKeysbyName[$name]=${projectKeysbyName[$name]}."
done
debug ${!F}_${!L} "----------------> END: getSonarQubeProjects"
} #getSonarQubeProjects
###################################################################################################
# runCurl
# launch curl and display the answer. Raise an error after a given duration...
# Argument: url ($1) output from curl ($2)
# example: runCurl "$URL/api/profiles/list?format=json" "/tmp/file.txt"
###################################################################################################
function runCurl
{
local myUrl=$1
local myOut=$2
debug ${!F}_${!L} "----------------> BEGIN: runCurl $myURL $myOut"
local retValue=1
local duration=0
local timeMax=30 # 30 s max !
SECONDS=0
# DEBUG: pay attention with debug messages (kept with pipes using runCurl)
while [ "$retValue" != "0" ]; do
rm $myOut 2>/dev/null
curl -ksf $myUrl > $myOut
retValue=$?
duration=$SECONDS
if [ "$duration" -ge "$timeMax" ]; then
error $LINENO "unable to get data from sonarQube : \"curl $myUrl\" : $(($duration % 60)) seconds elapsed (max is set to $timeMax s)."
retValue=0
fi
done
debug ${!F}_${!L} "----------------> END: runCurl"
}
###################################################################################################
# saveReportProject
# Argument: projet key ($1) output filename ($2)
# In Globals: SonarQube,projectQualityGatebyKey toolVersion
###################################################################################################
function saveReportProject
{
local projectKey=$1
local outFile=$2
debug ${!F}_${!L} "----------------> BEGIN: saveReportProject $projectKey $outFile"
local projectId=${projectIdbyKey[${projectKey}]}
local projectName=${projectNamebyId[$projectId]}
local lineSep="# $lineSep"
local URL=${SonarQube[URL]}
local tmpFile="/tmp/saveReportProject$$.txt"
local service=""
debug ${!F}_${!L} "projectKey=$projectKey"
debug ${!F}_${!L} "outFile=$outFile"
debug ${!F}_${!L} "projectId=$projectId"
[[ -f $outFile ]] && rm $outFile
echo $lineSep >> $outFile
echo "# Quality check performed on " $(date +"%x %r %Z") " by $USER" >> $outFile
echo -e "# SonarQube project name:$projectName\n# SonarQube Key:$projectKey\n# Last analysis in sonarQube:$URL/dashboard/index/$projectId" >> $outFile
runCurl "$URL/api/profiles/list?format=json" $tmpFile
cat $tmpFile | jq -jr '.[] | "# \(.language):\"\(.name)\" "' >> $outFile 2>/dev/null
echo -e "\n# SonarQube $URL Version:${SonarQube[version]}" >> $outFile
echo -e "# ${SonarQube[plugins]}" >> $outFile
echo "# gcc=${toolVersion[gcc]},cppcheck=${toolVersion[cppcheck]},verapp=${toolVersion[verapp]},valgrind=${toolVersion[valgrind]},gcovr=${toolVersion[gcovr]}" >> $outFile
echo "# python=${toolVersion[python]},pylint=${toolVersion[pylint]},coverage=${toolVersion[coverage]}" >> $outFile
echo "$lineSep" >> $outFile
echo "# Size" >> $outFile
runCurl "$URL/api/resources?resource=${projectKey}&metrics=files,directories,classes,ncloc,lines,functions,statements" $tmpFile
cat $tmpFile | jq -r '.[].msr[] | "\(.key) : \(.val)"' >> $outFile 2>/dev/null
echo $lineSep >> $outFile
echo "# Complexity" >> $outFile
runCurl "$URL/api/resources?resource=${projectKey}&metrics=complexity,file_complexity,class_complexity,function_complexity" $tmpFile
cat $tmpFile | jq -r '.[].msr[] | "\(.key) : \(.val)"' >> $outFile 2>/dev/null
echo $lineSep >> $outFile
echo "# Documentation" >> $outFile
runCurl "$URL/api/resources?resource=${projectKey}&metrics=comment_lines,comment_lines_density,public_documented_api_density,public_undocumented_api" $tmpFile
cat $tmpFile | jq -r '.[].msr[] | "\(.key) : \(.val)"' >> $outFile 2>/dev/null
echo $lineSep >> $outFile
echo "# Duplications" >> $outFile
runCurl "$URL/api/resources?resource=${projectKey}&metrics=duplicated_blocks,duplicated_files,duplicated_lines,duplicated_lines_density" $tmpFile
cat $tmpFile | jq -r '.[].msr[] | "\(.key) : \(.val)"' >> $outFile 2>/dev/null
echo $lineSep >> $outFile
echo "# Tests" >> $outFile
runCurl "$URL/api/resources?resource=${projectKey}&metrics=coverage,line_coverage,branch_coverage,tests,test_execution_time,test_errors,test_failures,test_success_density" $tmpFile
cat $tmpFile | jq -r '.[].msr[] | "\(.key) : \(.val)"' >> $outFile 2>/dev/null
echo $lineSep >> $outFile
echo "# Quality Gate" >> $outFile
runCurl "$URL/api/qualitygates/list" $tmpFile
qualitygates=$(cat $tmpFile | jq -jr '.qualitygates[] | "\(.name) " ' 2>/dev/null)
echo "Available Quality gates : $qualitygates" >> $outFile
echo "Project Quality gate : ${projectQualityGatebyKey[$projectId]}" >> $outFile
runCurl "$URL/api/resources?resource=${projectKey}&metrics=quality_gate_details" $tmpFile
cat $tmpFile | jq -r '.[].msr[].data' | jq -r '' | jq -r '. | "Quality Gate status: \(.level)"' >> $outFile 2>/dev/null
cat $tmpFile | jq -r '.[].msr[].data' | jq -r '' | jq -r '.conditions[] | "\(.metric): \(.level)"' >> $outFile 2>/dev/null
echo $lineSep >> $outFile
echo "# Issues" >> $outFile
runCurl "$URL/api/resources?resource=${projectKey}&metrics=violations,weighted_violations,violations_density,sqale_index,blocker_violations,critical_violations,major_violations,minor_violations,info_violations" $tmpFile
cat $tmpFile | jq -r '.[].msr[] | "\(.key) : \(.val)"' >> $outFile 2>/dev/null
local severities="MINOR,MAJOR,CRITICAL,BLOCKER"
runCurl "$URL/api/resources?resource=${projectKey}&metrics=minor_violations" $tmpFile
local nb_minor=$(cat $tmpFile | jq -r '.[].msr[] | "\(.val)"' 2>/dev/null)
if [ "${nb_minor}" -ge "100" ] ; then
severities="MAJOR,CRITICAL,BLOCKER"
fi
echo "# List of issues with severities $severities:" >> $outFile
echo $lineSep >> $outFile
echo -e "# Component\t\t\tLine\tRule\tSeverity\tDebt\tMessage" >> $outFile
echo $lineSep >> $outFile
# sonarqube 5.1
#curl -ks "$URL/api/issues/search?componentRoots=${projectKey}&severities=${severities}&resolved=false" | jq -r '.issues[] | "\(.component)\t\(.line)\t\(.rule)\t\(.severity)\t\(.debt)\t\(.message)"' | sort >> $outFile 2>/dev/null
#sonarqube 5.3
#TODO: how to cut easily the message (valgrind problem) ??
runCurl "$URL/api/issues/search?componentKeys=${projectName}&severities=${severities}&resolved=false" $tmpFile
cat $tmpFile | jq -r '.issues[] | "\(.component)\t\(.line)\t\(.rule)\t\(.severity)\t\(.debt)\t\(.message)"' | sort >> $outFile 2>/dev/null
echo $lineSep >> $outFile
rm $tmpFile 2>/dev/null
debug ${!F}_${!L} "----------------> END: saveReportProject"
} # saveReportProject
###################################################################################################
# test the internet connection and exit if no connection
# Argument: none
###################################################################################################
function exitIfNoInternet
{
debug ${!F}_${!L} "----------------> BEGIN: exitIfNoInternet"
#wget -q --spider http://google.com
ping -c 1 google.com &> /dev/null
if [ ! $? -eq 0 ]; then
error $LINENO "No internet connection"
fi
debug ${!F}_${!L} "----------------> END: exitIfNoInternet"
} # exitIfNoInternet
###################################################################################################
# test the sonarqube connection and exit if no connection
# Argument: none
# Global In/Out: SonarQube
###################################################################################################
function getSonarQubeInfos
{
debug ${!F}_${!L} "----------------> BEGIN: getSonarQubeInfos"
local tmpFile="/tmp/getSonarQubeInfos$$.txt"
runCurl "${SonarQube[URL]}/api/server/index" $tmpFile
local status=$(cat $tmpFile | jq -r '.status' 2>/dev/null)
local versionSonarQube=$(cat $tmpFile | jq -r '.version' 2>/dev/null)
debug ${!F}_${!L} "SonarQube ${SonarQube[URL]}:status=$status."
runCurl "${SonarQube[URL]}/api/updatecenter/installed_plugins" $tmpFile
local plugins=$(cat $tmpFile | jq -jr '.[] | "\(.key):\"\(.name)\":\(.version) "' 2>/dev/null)
SonarQube[status]=$status
SonarQube[version]=$versionSonarQube
SonarQube[plugins]=$plugins
debug ${!F}_${!L} "${SonarQube[URL]}: SonarQube[status]=${SonarQube[status]},SonarQube[version]=${SonarQube[version]}."
debug ${!F}_${!L} "SonarQube[plugins]=${SonarQube[plugins]}."
if [ "${SonarQube[status]}" != "UP" ]; then
error $LINENO "${SonarQube[URL]}: SonarQube status is not \"UP\": status is \"${SonarQube[status]}\""
fi
rm $tmpFile 2>/dev/null
debug ${!F}_${!L} "----------------> END: getSonarQubeInfos"
} # getSonarQubeInfos
###################################################################################################
# printSonarQubeProjects
# Argument: option: $1="codeen"
# IN Global: SonarQube_CODEEN
# Out Global: sonarQubeURL
###################################################################################################
function printSonarQubeProjects
{
local option=$1
debug ${!F}_${!L} "----------------> BEGIN: printSonarQubeProjects option=$option"
[[ "$option" == "codeen" ]] && SonarQube[URL]=$sonarQubeURL_CODEEN
[[ "$option" == "codeen" ]] && exitIfNoInternet
getSonarQubeInfos
getSonarQubeProjects
local format="\e[92m%-30s %s\e[0m\n"
echo "SonarQube URL:${SonarQube[URL]}"
printf "$format" "------------" "-----------------------"
printf "$format" "PROJECT NAME" "SONARQUBE PROJECTS KEYS"
printf "$format" "------------" "-----------------------"
for project in "${!projectKeysbyName[@]}"; do
printf "$format" "$project" "${projectKeysbyName[${project}]}"
done
debug ${!F}_${!L} "----------------> END: printSonarQubeProjects"
} # printSonarQubeProjects
###################################################################################################
# produceSonarQubeReport
# Argument: optional project key ($1) and option: $2=="codeen"
# IN Global: sonarQubeURL_CODEEN,reportsDir ${project[name]} ${project[sonarVersion]}
# Out Global: sonarQube sonarQube_reportFile
###################################################################################################
function produceSonarQubeReport
{
local projectKey=$1
local option=$2
debug ${!F}_${!L} "----------------> BEGIN:printSonarQubeReport projectKey=$projectKey,option=$option"
if [[ "$projectKey" == "" ]] ; then
get_project
projectKey="${project[name]}-${project[sonarVersion]}"
# error $LINENO "project key is required ! (try '> $scriptName -p $option' and choose a project key in the right column)"
fi
debug ${!F}_${!L} "projectKey=$projectKey"
[[ "$option" == "codeen" ]] && SonarQube[URL]=$sonarQubeURL_CODEEN
[[ "$option" == "codeen" ]] && exitIfNoInternet
getSonarQubeInfos
getSonarQubeProjects
getSonarQubeQualityGates
get_toolVersion
debug ${!F}_${!L} "projectIdbyKey[${projectKey}]=${projectIdbyKey[${projectKey}]}."
if [[ "${projectIdbyKey[${projectKey}]}" == "" ]] ; then
error $LINENO "project key \"$projectKey\" is unknown in ${SonarQube[URL]} (try '> $scriptName -p $option' and choose a project key in the right column)"
fi
sonarQube_reportFile="${reportsDir}/${projectKey}_report.txt"
saveReportProject $projectKey $sonarQube_reportFile
[[ ! -s "$sonarQube_reportFile" ]] && error $LINENO "sonarQube analysis summary \"$sonarQube_reportFile\" is empty or not found"
debug ${!F}_${!L} "----------------> END: produceSonarQubeReport"
} # produceSonarQubeReport
###################################################################################################
# displaySonarQubeReport
# Argument: optional project key ($1) and option: $2=="codeen"
# IN Global: sonarQubeURL_CODEEN,reportsDir
# Out Global: sonarQube sonarQube_reportFile
###################################################################################################
function displaySonarQubeReport
{
local projectKey=$1
local option=$2
debug ${!F}_${!L} "----------------> BEGIN: displaySonarQubeReport projectKey=$projectKey,option=$option"
local display="cat"
initLogbook append
if [[ "$projectKey" == "diff" ]] ; then
projectKey=""
display="vimdiff"
echo "highlight Normal term=none cterm=none ctermfg=White ctermbg=Black gui=none guifg=White guibg=Black" > ~/.vimrc
fi
produceSonarQubeReport $projectKey $option
if [[ "$display" == "vimdiff" ]] ; then
local out=$(basename $sonarQube_reportFile)
local sonarQube_previousreportFile="${reportsDir}/${reportsDir}_previous/$out"
if [[ -s "${sonarQube_previousreportFile}" ]] ; then
vimdiff $sonarQube_reportFile $sonarQube_previousreportFile
else
error $LINENO "previous sonarQube analysis summary \"${sonarQube_previousreportFile}\" is empty or not found"
fi
else
cat $sonarQube_reportFile
info "sonarQube analysis summary in \"$sonarQube_reportFile\""
fi
debug ${!F}_${!L} "----------------> END: displaySonarQubeReport"
} # displaySonarQubeReport
###################################################################################################
# displaySonarQubeSummary
# # Argument: project key ($1)
# In Global: sonarQube_reportFile
###################################################################################################
function displaySonarQubeSummary
{
local projectKey=$1
debug ${!F}_${!L} "----------------> BEGIN: displaySonarQubeSummary projectKey=$projectKey"
local tmpFile="/tmp/displaySonarQubeSummary$$.txt"
local reg='^[0-9]+$' # regular expression for an expected integer
local sleepInSeconds
sleep 2 # wait 2 seconds in any cases
info "SonarQube dashboard is available on ${SonarQube[URL]}/dashboard/?id=${sonar_projectKey}&did=1"
runCurl "${SonarQube[URL]}/api/resources?resource=${projectKey}&metrics=lines" $tmpFile
debug ${!F}_${!L} "runCurl \"${SonarQube[URL]}/api/resources?resource=${projectKey}&metrics=lines\" $tmpFile"
#local lines=$(cat $tmpFile | jq -r '.[].msr[] | "\(.key) : \(.val)"' 2>/dev/null)
local lines=$(cat $tmpFile | jq -r '.[].msr[] | "\(.val)"' 2>/dev/null)
debug ${!F}_${!L} "lines =${lines}."
if [ -z $lines ] ; then
debug ${!F}_${!L} "Variable lines is empty: no summary report produced"
produceSonarQubeReport ${projectKey}
elif ! [[ $lines =~ $reg ]] ; then
debug ${!F}_${!L} "Variable lines is not a integer: no summary report produced"
produceSonarQubeReport ${projectKey}
else
# wait 1 s each 5000 lines
sleepInSeconds=$(($lines / 5000))
debug ${!F}_${!L} "Wait for $sleepInSeconds seconds..."
sleep $sleepInSeconds
produceSonarQubeReport ${projectKey}
info "SonarQube analysis summary:"
grep "^lines :\|^tests :\|^line_coverage :\|.*_violations" $sonarQube_reportFile
info "More in \"$sonarQube_reportFile\""
fi
debug ${!F}_${!L} "----------------> END: displaySonarQubeSummary"
} # displaySonarQubeSummary
##################################################################
# display information environment
# In optional : if set to "tools" do no check current directory (i.e. only tools used)
# In Globals: linesep requiredEnv toolVersion requiredEnv
# project projectCodePaths projectUSE BINARY_TAG_cov
# cppBinTests pythonBinTests
##################################################################
function printEnvironment
{
local tools=$1
debug ${!F}_${!L} "----------------> BEGIN: printEnvironment : tools=$tools"
local analysis
local tool
local enVar
get_toolVersion # available tools
for analysis in static dynamic ; do
info $lineSep
info "Mandatory tools for a complete $analysis analysis:"