diff --git a/compare-flatcar-with-gentoo b/compare-flatcar-with-gentoo deleted file mode 100755 index 85cbd1f..0000000 --- a/compare-flatcar-with-gentoo +++ /dev/null @@ -1,809 +0,0 @@ -#!/bin/bash - -# This script tries to compare the package versions in portage-stable, -# coreos-overlay and gentoo. When done, prints some statistics to -# standard output. -# -# Example invocations: -# ==================== -# -# ./compare-flatcar-with-gentoo -# -# export COREOS_OVERLAY=../../co-repo -# export PORTAGE_STABLE=../../ps-repo -# export GENTOO=../../g-repo -# export JSON=x -# ./compare-flatcar-with-gentoo -# -# Environment variables: -# ====================== -# -# COREOS_OVERLAY: -# -# A path to the coreos-overlay git repository. -# -# PORTAGE_STABLE: -# -# A path to the portage-stable git repository. -# -# GENTOO: -# -# A path to the gentoo git repository. -# -# JSON: -# -# If not empty, print statistics in JSON format. -# -# Environment variables used for debugging: -# ========================================= -# -# WORKDIR: -# -# A path to a working directory that will hold temporary files. -# -# KEEP_WORKDIR: -# -# If specified to non-empty value, then the workdir will be printed to -# standard error if verbose output is enabled and will not be removed -# after script finishes. -# -# VERBOSE: -# -# Print debugging stuff to standard error. - -set -euo pipefail - -shopt -s extglob - -this_dir="$(dirname "${0}")" - -: ${COREOS_OVERLAY:="${this_dir}/../coreos-overlay"} -: ${PORTAGE_STABLE:="${this_dir}/../portage-stable"} -: ${GENTOO:="${this_dir}/../gentoo"} -: ${JSON:=} -: ${WORKDIR:="$(mktemp --directory "${this_dir}/cfwg.XXXXXXXXXX")"} -: ${KEEP_WORKDIR:=} -: ${VERBOSE:=} - -stderr() { - printf '%s\n' "$*" >/dev/stderr -} - -debug() { - if [[ -z "${VERBOSE}" ]]; then - return - fi - stderr "$@" -} - -warn() { - stderr 'WARNING:' "$@" -} - -drop_trailing_slash() { - local var_name="${1}" - - local -n var_ref="${var_name}" - - var_ref="${var_ref%%*(/)}" -} - -drop_trailing_slash PORTAGE_STABLE -drop_trailing_slash COREOS_OVERLAY -drop_trailing_slash GENTOO -drop_trailing_slash WORKDIR - -mkdir -p "${WORKDIR}" - -if [[ -z "${KEEP_WORKDIR}" ]]; then - trap "rm -rf '${WORKDIR}'" EXIT -else - debug "WORKDIR='${WORKDIR}'" -fi - -debug "PORTAGE_STABLE=${PORTAGE_STABLE}" -debug "COREOS_OVERLAY=${COREOS_OVERLAY}" -debug "GENTOO=${GENTOO}" - -prepend_with() { - local field_count="${1}" - local item="${2}" - local arr_name="${3}" - local -n arr_ref="${arr_name}" - - while [[ "${#arr_ref[@]}" -lt "${field_count}" ]]; do - arr_ref=("${item}" "${arr_ref[@]}") - done -} - -append_with() { - local field_count="${1}" - local item="${2}" - local arr_name="${3}" - local -n arr_ref="${arr_name}" - - while [[ "${#arr_ref[@]}" -lt "${field_count}" ]]; do - arr_ref+=("${item}") - done -} - -strip_leading_zeros() { - local num_name="${1}" - - local -n num_ref="${num_name}" - - # strip all leading zeros, will work only if extglob is enabled - num_ref="${num_ref##*(0)}" - - if [[ -z "${num_ref}" ]]; then - num_ref='0' - fi -} - -get_plain_version() { - local ver="${1}" - local arr_name="${2}" - - # strip everything after first underline or dash, whichever comes - # first, so for strings like 1.2.3_alpha or 1.2.3-r2 we get 1.2.3 - ver="${ver%%[-_]*}" - IFS='.' read -ra "${arr_name}" <<< "${ver}" - - local i - local -n arr_ref="${arr_name}" - - for i in $(seq 0 "$((${#arr_ref[@]}-1))"); do - strip_leading_zeros "${arr_name}[${i}]" - done -} - -# last version field may have a letter -split_last_version_field() { - local field="${1}" - local num_name="${2}" - local word_name="${3}" - - local -n num_ref="${num_name}" - local -n word_ref="${word_name}" - - num_ref="${field%%*([a-z])}" - word_ref="${field##*([0-9])}" - - strip_leading_zeros "${num_name}" -} - -get_suffixes() { - local ver="${1}" - local arr_name="${2}" - local ver_tmp - - if [[ ! "${ver}" =~ _ ]]; then - local -n arr_ref="${arr_name}" - arr_ref=() - return - fi - ver_tmp="${ver#*_}" - ver="${ver_tmp%-*}" - IFS='_' read -ra "${arr_name}" <<< "${ver}" -} - -split_suffix() { - local suffix="${1}" - local word_name="${2}" - local num_name="${3}" - - local -n word_ref="${word_name}" - local -n num_ref="${num_name}" - - word_ref="${suffix%%*([0-9])}" - num_ref="${suffix##*([a-z])}" - - strip_leading_zeros "${num_name}" -} - -get_revision() { - local version="${1}" - local rev_name="${2}" - - local -n rev_ref="${rev_name}" - - if [[ ! "${version}" =~ '-' ]]; then - rev_ref='0' - return - fi - rev_ref="${version##*-r}" - - strip_leading_zeros "${rev_name}" -} - -declare -A SUFFIXES=( - ['alpha']='0' - ['beta']='1' - ['pre']=2 - ['rc']=3 - # empty string can't be a key - # ['']=4 - ['p']=5 -) - -get_suffix_word_index() { - local word="${1}" - local index_ref="${2}" - - local -n index="${index_ref}" - - # we can't put an empty key into the hash map - if [[ -z "${word}" ]]; then - index=4 - return - fi - # assign -1 to invalid suffix - index="${SUFFIXES[${word}]--1}" -} - -gentoo_vercmp() { - local a_ver="${1}" - local b_ver="${2}" - local res_name="${3}" - - local -n res_ref="${res_name}" - - local -a a_nums - local -a b_nums - local a_last_field_idx - local b_last_field_idx - - get_plain_version "${a_ver}" a_nums - get_plain_version "${b_ver}" b_nums - a_last_field_idx="$((${#a_nums[@]}-1))" - b_last_field_idx="$((${#b_nums[@]}-1))" - - if [[ "${#a_nums[@]}" -ne "${#b_nums[@]}" ]]; then - if [[ "${#a_nums[@]}" -gt "${#b_nums[@]}" ]]; then - append_with "${#a_nums[@]}" 0 b_nums - else - append_with "${#b_nums[@]}" 0 a_nums - fi - fi - - local i - local a - local b - local a_num - local a_word - local b_num - local b_word - - for i in $(seq 0 $(("${#a_nums[@]}" - 1))); do - a="${a_nums[${i}]}" - b="${b_nums[${i}]}" - - if [[ "${i}" -eq "${a_last_field_idx}" ]]; then - split_last_version_field "${a}" a_num a_word - else - a_num="${a}" - a_word='' - if [[ ! "${a_num}" =~ [0-9] ]]; then - warn "Invalid version field '${a_num}' in '${a_ver}', only last version field can contain letters." - fi - fi - if [[ "${i}" -eq "${b_last_field_idx}" ]]; then - split_last_version_field "${b}" b_num b_word - else - b_num="${b}" - b_word='' - if [[ ! "${b_num}" =~ [0-9] ]]; then - warn "Invalid version field '${b_num}' in '${b_ver}', only last version field can contain letters." - fi - fi - - if [[ "${a_num}" -gt "${b_num}" ]]; then - res_ref=">v${i}n" - return - fi - if [[ "${a_num}" -lt "${b_num}" ]]; then - res_ref=" "${b_word}" ]]; then - res_ref=">v${i}w" - return - fi - if [[ "${a_word}" < "${b_word}" ]]; then - res_ref=""${WORKDIR}/${name}-pkgs" - git -C "${repo}" log -1 --pretty='%H' >"${WORKDIR}/${name}-hash" - git -C "${repo}" log -1 --pretty='%cD' >"${WORKDIR}/${name}-date" - fi - done - - cat "${WORKDIR}/coreos-overlay-pkgs" "${WORKDIR}/portage-stable-pkgs" | sort -u >"${WORKDIR}/flatcar-pkgs" - fi - - debug "finding all duplicated packages in portage-stable and coreos-overlay" - comm -1 -2 "${WORKDIR}/coreos-overlay-pkgs" "${WORKDIR}/portage-stable-pkgs" >"${WORKDIR}/duplicated-pkgs" - - debug "finding all flatcar-only packages in portage-stable" - comm -2 -3 "${WORKDIR}/portage-stable-pkgs" "${WORKDIR}/gentoo-pkgs" >"${WORKDIR}/portage-stable-flatcar-only-pkgs" - debug "finding all flatcar-only packages in coreos-overlay" - comm -2 -3 "${WORKDIR}/coreos-overlay-pkgs" "${WORKDIR}/gentoo-pkgs" >"${WORKDIR}/coreos-overlay-flatcar-only-pkgs" - debug "finding all flatcar-only packages" - cat "${WORKDIR}/portage-stable-flatcar-only-pkgs" "${WORKDIR}/coreos-overlay-flatcar-only-pkgs" | sort -u >"${WORKDIR}/flatcar-only-pkgs" - - debug "finding all common packages in portage-stable" - comm -1 -2 "${WORKDIR}/portage-stable-pkgs" "${WORKDIR}/gentoo-pkgs" >"${WORKDIR}/portage-stable-common-pkgs" - debug "finding all common packages in coreos-overlay" - comm -1 -2 "${WORKDIR}/coreos-overlay-pkgs" "${WORKDIR}/gentoo-pkgs" >"${WORKDIR}/coreos-overlay-common-pkgs" - debug "finding all common packages" - cat "${WORKDIR}/coreos-overlay-common-pkgs" "${WORKDIR}/portage-stable-common-pkgs" | sort -u >"${WORKDIR}/common-pkgs" -fi - -get_pkg_name() { - local name="${1}" - echo -n "${name#*/}" -} - -get_max_version() { - local pkg="${1}" - local max_version_name="${2}" - - shift 2 - - local repo - local pkg_path_prefix - local -a ebuilds - local e - local version - local result - local -n max_version_ref="${max_version_name}" - - # this is quite a low version - max_version_ref='0_alpha_alpha_alpha' - - for repo; do - if [[ ! -d "${repo}/${pkg}" ]]; then - continue - fi - - pkg_path_prefix="${repo}/${pkg}/$(get_pkg_name "${pkg}")-" - ebuilds=( "${pkg_path_prefix}"*.ebuild ) - - for e in "${ebuilds[@]}"; do - # strip prefix - version="$(cut_leading_path "${pkg_path_prefix}" <<<"${e}")" - # strip .ebuild - version="${version:0:-7}" - # ignore versions like 9999 or 99999999 - if [[ -z "${version//9}" ]]; then - continue - fi - result='=' - gentoo_vercmp "${version}" "${max_version_ref}" result - if [[ "${result}" = '>'* ]]; then - max_version_ref="${version}" - fi - done - return - done -} - -find_max_versions() { - local input="${1}" - local output="${2}" - - # create the file in case of an empty output - touch "${output}" - - while read -r line; do - debug " - ${line}" - gentoo_max='' - get_max_version "${line}" gentoo_max "${GENTOO}" - flatcar_max='' - get_max_version "${line}" flatcar_max "${PORTAGE_STABLE}" "${COREOS_OVERLAY}" - result='=' - gentoo_vercmp "${gentoo_max}" "${flatcar_max}" result - printf '%s %s\n' "${line}" "gentoo:${gentoo_max} flatcar:${flatcar_max} result:${result}" >>"${output}" - done < "${input}" -} - -if [[ ! -e "${WORKDIR}/portage-stable-common-pkgs-max-versions" ]]; then - debug "finding max versions of common packages in portage-stable" - find_max_versions "${WORKDIR}/portage-stable-common-pkgs" "${WORKDIR}/portage-stable-common-pkgs-max-versions" -fi - -if [[ ! -e "${WORKDIR}/coreos-overlay-common-pkgs-max-versions" ]]; then - debug "finding max versions of common packages in coreos-overlay" - find_max_versions "${WORKDIR}/coreos-overlay-common-pkgs" "${WORKDIR}/coreos-overlay-common-pkgs-max-versions" -fi - -if [[ ! -e "${WORKDIR}/common-pkgs-max-versions" ]]; then - debug "finding max versions of common packages" - cat "${WORKDIR}/portage-stable-common-pkgs-max-versions" "${WORKDIR}/coreos-overlay-common-pkgs-max-versions" | sort -u >"${WORKDIR}/common-pkgs-max-versions" -fi - -debug "done" - -percent() { - local num="${1}" - local den="${2}" - - bc <<< "scale=2; (${num} * 100) / ${den}" -} - -duplicates="$(cat "${WORKDIR}/duplicated-pkgs" | wc -l)" - -if [[ "${duplicates}" -gt 0 ]]; then - warn 'There are duplicated packages in portage-stable and coreos-overlay.' - warn 'Some stats may be skewed or make no sense.' - warn 'BEGIN DUPLICATES' - cat "${WORKDIR}/duplicated-pkgs" >/dev/stderr - warn 'END DUPLICATES' -fi - -# non-failing grep -nfgrep() { - grep "$@" || : -} - -ps_hash="$(cat "${WORKDIR}/portage-stable-hash")" -ps_date="$(cat "${WORKDIR}/portage-stable-date")" -co_hash="$(cat "${WORKDIR}/coreos-overlay-hash")" -co_date="$(cat "${WORKDIR}/coreos-overlay-date")" -gentoo_hash="$(cat "${WORKDIR}/gentoo-hash")" -gentoo_date="$(cat "${WORKDIR}/gentoo-date")" - -total="$(cat "${WORKDIR}/flatcar-pkgs" | wc -l)" -ps_pkgs="$(cat "${WORKDIR}/portage-stable-pkgs" | wc -l)" -co_pkgs="$(cat "${WORKDIR}/coreos-overlay-pkgs" | wc -l)" -ours="$(cat "${WORKDIR}/flatcar-only-pkgs" | wc -l)" -total_common="$(cat "${WORKDIR}/common-pkgs" | wc -l)" -ps_common="$(cat "${WORKDIR}/portage-stable-common-pkgs" | wc -l)" -co_common="$(cat "${WORKDIR}/coreos-overlay-common-pkgs" | wc -l)" -newer="$(nfgrep -e 'result:<' "${WORKDIR}/common-pkgs-max-versions" | wc -l)" -equal="$(nfgrep -e 'result:=' "${WORKDIR}/common-pkgs-max-versions" | wc -l)" -outdated="$(nfgrep -e 'result:>' "${WORKDIR}/common-pkgs-max-versions" | wc -l)" -outdated_rev="$(nfgrep -e 'result:>r' "${WORKDIR}/common-pkgs-max-versions" | wc -l)" -outdated_suf="$(nfgrep -e 'result:>s' "${WORKDIR}/common-pkgs-max-versions" | wc -l)" -outdated_ver="$(nfgrep -e 'result:>v' "${WORKDIR}/common-pkgs-max-versions" | wc -l)" - -ps_newer="$(nfgrep -e 'result:<' "${WORKDIR}/portage-stable-common-pkgs-max-versions" | wc -l)" -ps_equal="$(nfgrep -e 'result:=' "${WORKDIR}/portage-stable-common-pkgs-max-versions" | wc -l)" -ps_outdated="$(nfgrep -e 'result:>' "${WORKDIR}/portage-stable-common-pkgs-max-versions" | wc -l)" -ps_outdated_rev="$(nfgrep -e 'result:>r' "${WORKDIR}/portage-stable-common-pkgs-max-versions" | wc -l)" -ps_outdated_suf="$(nfgrep -e 'result:>s' "${WORKDIR}/portage-stable-common-pkgs-max-versions" | wc -l)" -ps_outdated_ver="$(nfgrep -e 'result:>v' "${WORKDIR}/portage-stable-common-pkgs-max-versions" | wc -l)" - -co_newer="$(nfgrep -e 'result:<' "${WORKDIR}/coreos-overlay-common-pkgs-max-versions" | wc -l)" -co_equal="$(nfgrep -e 'result:=' "${WORKDIR}/coreos-overlay-common-pkgs-max-versions" | wc -l)" -co_outdated="$(nfgrep -e 'result:>' "${WORKDIR}/coreos-overlay-common-pkgs-max-versions" | wc -l)" -co_outdated_rev="$(nfgrep -e 'result:>r' "${WORKDIR}/coreos-overlay-common-pkgs-max-versions" | wc -l)" -co_outdated_suf="$(nfgrep -e 'result:>s' "${WORKDIR}/coreos-overlay-common-pkgs-max-versions" | wc -l)" -co_outdated_ver="$(nfgrep -e 'result:>v' "${WORKDIR}/coreos-overlay-common-pkgs-max-versions" | wc -l)" - -out_type='spaces' -if [[ -n "${JSON}" ]]; then - out_type='json' -fi - -out_init() { - "${out_type}_out_init" "${@}" -} - -out_start() { - "${out_type}_out_start" "${@}" -} - -out_kv() { - "${out_type}_out_kv" "${@}" -} - -out_end() { - "${out_type}_out_end" "${@}" -} - -out_fini() { - "${out_type}_out_fini" "${@}" -} - -spaces_indent=' ' -spaces_first_len=40 -spaces_second_len=8 -spaces_first_group_start=1 - -spaces_out_init() { - : -} - -spaces_out_start() { - if [[ ${spaces_first_group_start} -eq 0 ]]; then - printf '\n'; - else - spaces_first_group_start=0 - fi - printf '%s:\n' "$*" -} - -spaces_repeat() { - local var_name="${1}" - local count="${2}" - local -n var_ref="${var_name}" - local str="$(printf '=%.0s' $(seq 1 "${count}"))" - - var_ref="${str//=/ }" -} - -spaces_out_kv() { - local k="${1}" - local t="${2}" - local v="${3}" - local kl="${#k}" - local ktabl=$((spaces_first_len-kl-1)) - local ktab - - spaces_repeat ktab "${ktabl}" - - if [[ "${#}" -gt 3 ]]; then - local vl="${#v}" - local vtabl=$((spaces_second_len-vl)) - local vtab - local p="${4}" - - spaces_repeat vtab "${vtabl}" - - printf '%s%s:%s%s%s(%s%%)\n' "${spaces_indent}" "${k}" "${ktab}" "${v}" "${vtab}" "${p}" - else - printf '%s%s:%s%s\n' "${spaces_indent}" "${k}" "${ktab}" "${v}" - fi -} - -spaces_out_end() { - : -} - -spaces_out_fini() { - : -} - -declare -a json_out_array - -json_drop_comma() { - local l="${#json_out_array[@]}" - - if [[ "${l}" -eq 0 ]]; then - return - fi - - local i=$((l - 1)) - - json_out_array[${i}]="${json_out_array[${i}]%,}" -} - -json_out_init() { - json_out_array=('{') -} - -json_out_start() { - json_out_array+=(" \"${*}\": {") -} - -json_escape_string() { - local var_name="${1}" - local str="${2}" - local -n var_ref="${var_name}" - # escape backslashes - local escaped="${str//\\/\\\\}" - # escape double quotes - var_ref="\"${escaped//\"/\\\"}\"" -} - -json_v() { - local var_name="${1}" - local t="${2}" - local v="${3}" - local -n var_ref="${var_name}" - - case "${t}" in - i|f) - var_ref="${v}" - ;; - *) - json_escape_string "${var_name}" "${v}" - ;; - esac -} - -json_out_kv() { - local k="${1}" - local t="${2}" - local v="${3}" - local pv - - json_v pv "${t}" "${v}" - json_out_array+=(" \"${k}\": ${pv},") -} - -json_out_end() { - json_drop_comma - json_out_array+=(' },') -} - -json_out_fini() { - json_drop_comma - json_out_array+=('}') - printf '%s\n' "${json_out_array[@]}" -} - -out_init - -out_start 'meta' -out_kv 'portage-stable hash' s "${ps_hash}" -out_kv 'portage-stable date' s "${ps_date}" -out_kv 'coreos-overlay hash' s "${co_hash}" -out_kv 'coreos-overlay date' s "${co_date}" -out_kv 'gentoo hash' s "${gentoo_hash}" -out_kv 'gentoo date' s "${gentoo_date}" -out_end - -out_start 'general' -out_kv 'total packages' i "${total}" -out_kv 'portage-stable' i "${ps_pkgs}" "$(percent "${ps_pkgs}" "${total}")" -out_kv 'coreos-overlay' i "${co_pkgs}" "$(percent "${co_pkgs}" "${total}")" -out_kv 'our packages' i "${ours}" "$(percent "${ours}" "${total}")" -out_kv 'common packages' i "${total_common}" "$(percent "${total_common}" "${total}")" -out_kv 'common packages in portage-stable' i "${ps_common}" "$(percent "${ps_common}" "${total}")" -out_kv 'common packages in coreos-overlay' i "${co_common}" "$(percent "${co_common}" "${total}")" -out_kv 'newer packages' i "${newer}" "$(percent "${newer}" "${total}")" -out_kv 'equal packages' i "${equal}" "$(percent "${equal}" "${total}")" -out_kv 'outdated packages' i "${outdated}" "$(percent "${outdated}" "${total}")" -out_kv 'outdated revision packages' i "${outdated_rev}" "$(percent "${outdated_rev}" "${total}")" -out_kv 'outdated suffix packages' i "${outdated_suf}" "$(percent "${outdated_suf}" "${total}")" -out_kv 'outdated version packages' i "${outdated_ver}" "$(percent "${outdated_ver}" "${total}")" -out_end - -out_start 'portage-stable' -out_kv 'total packages' i "${ps_pkgs}" -out_kv 'common packages' i "${ps_common}" "$(percent "${ps_common}" "${ps_pkgs}")" -out_kv 'newer packages' i "${ps_newer}" "$(percent "${ps_newer}" "${ps_pkgs}")" -out_kv 'equal packages' i "${ps_equal}" "$(percent "${ps_equal}" "${ps_pkgs}")" -out_kv 'all outdated packages' i "${ps_outdated}" "$(percent "${ps_outdated}" "${ps_pkgs}")" -out_kv 'outdated revision packages' i "${ps_outdated_rev}" "$(percent "${ps_outdated_rev}" "${ps_pkgs}")" -out_kv 'outdated suffix packages' i "${ps_outdated_suf}" "$(percent "${ps_outdated_suf}" "${ps_pkgs}")" -out_kv 'outdated version packages' i "${ps_outdated_ver}" "$(percent "${ps_outdated_ver}" "${ps_pkgs}")" -out_end - -out_start 'coreos-overlay' -out_kv 'total packages' i "${co_pkgs}" -out_kv 'common packages' i "${co_common}" "$(percent "${co_common}" "${co_pkgs}")" -out_kv 'newer packages' i "${co_newer}" "$(percent "${co_newer}" "${co_pkgs}")" -out_kv 'equal packages' i "${co_equal}" "$(percent "${co_equal}" "${co_pkgs}")" -out_kv 'all outdated packages' i "${co_outdated}" "$(percent "${co_outdated}" "${co_pkgs}")" -out_kv 'outdated revision packages' i "${co_outdated_rev}" "$(percent "${co_outdated_rev}" "${co_pkgs}")" -out_kv 'outdated suffix packages' i "${co_outdated_suf}" "$(percent "${co_outdated_suf}" "${co_pkgs}")" -out_kv 'outdated version packages' i "${co_outdated_ver}" "$(percent "${co_outdated_ver}" "${co_pkgs}")" -out_end - -out_fini diff --git a/coreos-overlay-diff.py b/coreos-overlay-diff.py deleted file mode 100755 index 35aeecb..0000000 --- a/coreos-overlay-diff.py +++ /dev/null @@ -1,195 +0,0 @@ -#!/usr/bin/env python3 - -from sh import which, grep, cut, mktemp, ErrorReturnCode # Requires "sh": sudo dnf install python3-sh -from sh.contrib import git -# Read docs here: http://amoffat.github.io/sh/index.html - -import argparse -import os -from pathlib import Path - -parser = argparse.ArgumentParser(description="Compare two coreos-overlay branches including " - "dereferened CROS_WORKON_COMMIT branches of " - "repositories located in coreos-overlay/../.") - -parser.add_argument("THEIRS", type=str, help="Reference/branch to compare to") -parser.add_argument("--ours", type=str, help="Our reference (defaults to \"HEAD\")") -parser.add_argument("--coreos-overlay", type=str, help="Path to coreos-overlay repository (defaults to \".\")") -parser.add_argument("--no-color", dest="no_color", action="store_true", help="Don't pipe diff through colordiff") -parser.add_argument("--no-commits", dest="no_commits", action="store_true", help="Don't log which commits are missing") -parser.add_argument("--no-diffs", dest="no_diffs", action="store_true", help="Don't show diffs") -parser.add_argument("--no-prs", dest="no_prs", action="store_true", help="Don't show PR links for merge commits " - "(ignoring cherry picks without a merge commit as cherry-pick-for can do)") -parser.add_argument("--diff-style", choices=["standard", "word-diff", "icdiff", "colordiff", "delta"], - help="Instead of standard git diff, use either git --word-diff, git icdiff, or pipe the existing diff through colordiff or delta") -parser.set_defaults(ours="HEAD", coreos_overlay=".", no_color=False, no_commits=False, no_diffs=False) -args = parser.parse_args() - -base_folder = str(Path(args.coreos_overlay + "/../").resolve()) + "/" - -if args.diff_style == "colordiff": - if not which("colordiff"): - raise Exception("colordiff not installed, try to run: sudo dnf install colordiff") - from sh import colordiff -elif args.diff_style == "delta": - if not which("delta"): - raise Exception("delta not installed, try to run: cargo install --git https://github.com/dandavison/delta") - from sh import delta - colordiff = delta.bake("--theme=none", "--color-only", "--paging=never") - -warnings = [] - -repo_map = {"coreos-init": "init", "cros-devutils": "flatcar-dev-util", "gmerge": "flatcar-dev-util", - "fero-client": "fero", "actool": "spec"} - - -def commits_to_pick(src="src", dst="dst"): - # Shows which commits in src can be picked from src to dst. - # The output is in the format of git log dst..src but excluding - # changes to .github and commits that are cherry-picked already. - # The function is equivalent to these two commands but with error checking: - # $ git cherry dst src | grep ^+ | cut -d " " -f 2 > tmp_outfile - # $ git log --no-merges --cherry-pick --format=%H dst..src -- . :!.github | grep -F -x -f tmp_outfile | xargs git show -s - # In our case --cherry-pick doesn't actually filter out we want to filter out - # (maybe a Flatcar branch strangeness), so we need to postprocess with git cherry. - tmp_outfile = str(mktemp("-u")).split("\n")[0] - commits_src_has_with_cherry = git.log("--no-merges", "--cherry-pick", # set --cherry-pick just in case it will filter out something already - "--format=%H", dst + ".." + src , "--", ".", ":!.github") - _commits_src_has_without_cherry = cut(grep(git.cherry(dst, src, _bg=False), - "^+", _ok_code=[1, 0]), - "-d", " ", "-f", "2", _out=tmp_outfile) - commits_src_has_filtered = str(grep(commits_src_has_with_cherry, - "-F", "-x", "-f", tmp_outfile, - _ok_code=[1, 0])).strip().split("\n") - commits_src_has_filtered = [commit for commit in commits_src_has_filtered if commit != ""] - if len(commits_src_has_filtered) > 0: - git_log = git.show("-s", *commits_src_has_filtered) - else: - git_log = "" - os.remove(tmp_outfile) - return git_log - -def pull_requests_for_merge_commits(src="src", dst="dst", repo="repo"): - # Prints links for GitHub PRs from merge commits. - # It uses merge commits which means that we don't find the PRs for cherry picks unless they have a merge commit - # as the cherry-pick-for script can do. - # The git log format is "subject#body" being "Merge pull request #NUMBER from BRANCH#PR_TITLE". - merge_commits = str(git.log("--merges", "--format=%s#%b", dst + ".." + src)) - filtered = [line for line in merge_commits.split("\n") if "Merge pull request" in line and line.count("#") >= 2] - # Won't panic because we ensured above that two # characters exist - pr_and_titles = [(line.split("#")[1].split(" ")[0], "#".join(line.split("#")[2:])) for line in filtered] - # TODO: find the correct upstream GitHub organization (from branch?) if it isn't kinvolk but systemd or coreos - # Ignores PRs that tell that they only change the .github folder by having a title starting with ".github" - links = [title + ": https://github.com/kinvolk/" + repo + "/pull/" + pr for (pr, title) in pr_and_titles if not title.startswith(".github")] - return "\n".join(links) - -def display_difference(from_theirs, to_ours, name, recurse=False): - # That means, show what "our" branch adds to "their" branch - diff_args = [from_theirs + ".." + to_ours, "--", ".", ":!.github"] - diff = git.diff(*diff_args, _bg=False, _decode_errors="replace") - desc_start = "↓" * 25 - desc_end = "↑" * 25 - desc = "Diff for " + name - if not args.no_diffs: - print(desc_start, desc, desc_start + "\n") - if args.diff_style == "icdiff": - print(git.difftool("-y", "-x", "icdiff --is-git-diff --cols=160", *diff_args, _bg=False, _decode_errors="replace")) - elif args.diff_style == "word-diff": - print(git.diff("--word-diff", "--no-color" if args.no_color else "--color", *diff_args, _bg=False, _decode_errors="replace")) - elif args.no_color: - print(diff) - elif args.diff_style == "colordiff" or args.diff_style == "delta": - print(colordiff(diff, _decode_errors="replace")) - else: - print(git.diff("--color", *diff_args, _bg=False, _decode_errors="replace")) - print("\n" + desc_end, desc, desc_end + "\n") - if not args.no_commits: - # Branch "our" is the checked out branch and "git diff" shows what "our" branch has that "their" branch doesn't. - # Converting the diff to "our" commits to pick for "their" branch) means setting "src" as "our". - commits_we_have = commits_to_pick(src=to_ours, dst=from_theirs) - commits_they_have = commits_to_pick(src=from_theirs, dst=to_ours) - desc = "Commits for " + name + " in our " + to_ours + " but not in their " + from_theirs - print(desc_start, desc, desc_start + "\n") - print(commits_we_have) - print("\n" + desc_end, desc, desc_end + "\n") - desc = "Commits for " + name + " in their " + from_theirs + " but not in our " + to_ours - print(desc_start, desc, desc_start + "\n") - print(commits_they_have) - print("\n" + desc_end, desc, desc_end + "\n") - if not args.no_prs: - prs_we_have = pull_requests_for_merge_commits(src=to_ours, dst=from_theirs, repo=name) - prs_they_have = pull_requests_for_merge_commits(src=from_theirs, dst=to_ours, repo=name) - desc = "PRs (from merge commits) for " + name + " in our " + to_ours + " but not in their " + from_theirs - print(desc_start, desc, desc_start + "\n") - print(prs_we_have) - print("\n" + desc_end, desc, desc_end + "\n") - desc = "PRs (from merge commits) for " + name + " in their " + from_theirs + " but not in our " + to_ours - print(desc_start, desc, desc_start + "\n") - print(prs_they_have) - print("\n" + desc_end, desc, desc_end + "\n") - if recurse: - theirs = "" - ours = "" - repo = "" - for line in diff.splitlines(): - if line.startswith("diff --git ") and line.endswith("ebuild"): - if theirs != "" or ours != "": - warnings.append("Error: Unexpected variable content (theirs: [expected empty]: " + theirs + ", ours [expected empty]: " + ours + ") for " + repo) - theirs = "" - ours = "" - repo = "-".join(line.split("/")[-1].split("-")[:-1]) # Get "some-name" for ".../some-name-9999.ebuild" - if repo in repo_map: - repo = repo_map[repo] - # @TODO: Add DOCKER_GITCOMMIT, COMMIT_ID, CONTAINERD_COMMIT - if "CROS_WORKON_COMMIT=" in line: - if repo == "": - raise Exception("No repo seen for: " + line) - is_theirs = line.startswith("-") - is_ours = line.startswith("+") - if not is_theirs and not is_ours: - raise Exception("Unexpected line:" + line) - # Checks that "- ..." is followed by "+ ..." - if is_theirs: - if theirs != "" or ours != "": - warnings.append("Error: Unexpected variable content, expected empty (theirs: " + theirs + ", ours: " + ours + ") for " + repo + " in: " + line) - theirs = "" - ours = "" - repo = "" - continue - theirs = line.split("\"")[1] - if is_ours: - if theirs == "" or ours != "": - warnings.append("Error: Unexpected variable content (theirs [expected not empty]: " + theirs + ", ours [expected empty]: " + ours + ") for " + repo + " in: " + line) - theirs = "" - ours = "" - repo = "" - continue - ours = line.split("\"")[1] - if theirs != "" and ours != "": - os.chdir(base_folder) - try: - os.chdir(base_folder + repo) - except FileNotFoundError: - print("Failed to enter repo directory for \"" + repo + "\", trying to clone it") - git.clone("git@github.com:kinvolk/" + repo + ".git") - os.chdir(repo) - try: - git.fetch("github") - except ErrorReturnCode: - print("Tried to fetch from github without success, trying to fetch the default remote.") - git.fetch() - print(desc_start, "Difference for", repo, desc_start + "\n") - display_difference(theirs, ours, repo) - print("\n" + desc_end, "Difference for", repo, desc_end + "\n") - repo = "" - theirs = "" - ours = "" - - -os.chdir(args.coreos_overlay) -display_difference(args.THEIRS, args.ours, os.path.basename(os.path.abspath(".")), recurse=True) -if warnings: - print("Encountered some errors when trying to compare recursively, probably due to deleted files:") - print("\n".join(warnings)) - print() -print("Done. Displayed all differences.") diff --git a/coreos-overlay-diff.py b/coreos-overlay-diff.py new file mode 120000 index 0000000..47c96f8 --- /dev/null +++ b/coreos-overlay-diff.py @@ -0,0 +1 @@ +flatcar-overlay-diff.py \ No newline at end of file diff --git a/create-manifest b/create-manifest deleted file mode 100755 index 0a510a8..0000000 --- a/create-manifest +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash - -SCRIPTFOLDER="$(dirname "$(readlink -f "$0")")" - -if [ "$VERSION" = "" ] || [ "$SDK_VERSION" = "" ] || [ "$CHANNEL" = "" ] || [ "$1" = "-h" ] || [ "$1" = "--help" ]; then - echo "$0:" - echo "This script will create a branch in the manifest repository checked out in $SCRIPTFOLDER/../manifest and push the branch and tag." - echo "The repository is cloned from github.com/kinvolk/manifest.git if it does not exist (origin in an existing repository must point there" - echo "or pushing fails or does some unwanted action)." - echo "Set VERSION, SDK_VERSION, and CHANNEL as environment variables, e.g., VERSION=2345.3.0 SDK_VERSION=2345.2.0 CHANNEL=stable $0" - echo - echo "It creates tags only for the manifest repository. Tags for the repositories scripts, coreos-overlay, and portage-stable should be" - echo "created first with ./tag-release (which can be rerun on build failures to introduce fixes without changing the manifest)." - echo "By default the manifest references refs/tags/CHANNEL-VERSION for the repositories scripts, coreos-overlay, and portage-stable" - echo "and simply refs/heads/master for the others because their revisions are defined by the CROS_WORKON_COMMIT in the respective ebuild files." - echo "Set the environment variables SCRIPTS_REF, OVERLAY_REF, PORTAGE_REF, or DEFAULT_REF to specify any other reference." - exit 1 -fi - -set -euo pipefail - -SCRIPTS_REF="${SCRIPTS_REF-refs/tags/$CHANNEL-$VERSION}" -OVERLAY_REF="${OVERLAY_REF-refs/tags/$CHANNEL-$VERSION}" -PORTAGE_REF="${PORTAGE_REF-refs/tags/$CHANNEL-$VERSION}" -DEFAULT_REF="${DEFAULT_REF-refs/heads/master}" - -echo "Running with CHANNEL=$CHANNEL VERSION=$VERSION SDK_VERSION=$SDK_VERSION" -echo "SCRIPTS_REF=$SCRIPTS_REF OVERLAY_REF=$OVERLAY_REF PORTAGE_REF=$PORTAGE_REF" -echo "DEFAULT_REF=$DEFAULT_REF" - -cd "$SCRIPTFOLDER/.." - -MANIFESTFOLDER="manifest" - -if [ -d "${MANIFESTFOLDER}" ]; then - git -C "${MANIFESTFOLDER}" fetch --prune origin -else - git clone git@github.com:kinvolk/manifest.git "${MANIFESTFOLDER}" -fi - -cd "${MANIFESTFOLDER}" - -if [ "$(git status --porcelain || echo failed)" != "" ]; then - echo "Error: Unexpected output of git status:" - git status - exit 1 -fi - -BRANCHNAME="flatcar-$CHANNEL-$VERSION" -EXISTS=0 -echo "Preparing branch" -git checkout flatcar-master -git pull -git branch "$BRANCHNAME" || EXISTS=1 -git checkout "$BRANCHNAME" -if [ "$EXISTS" = 1 ]; then - echo "Warning: Reusing existing branch $BRANCHNAME, will try to pull." - git pull || echo "Warning: Pulling failed. Ignore the above output if the branch just exists locally." -fi -echo "Preparing files" -sed -E -i "s/(FLATCAR_VERSION=)(.*)/\1$VERSION/" version.txt -sed -E -i "s/(FLATCAR_VERSION_ID=)(.*)/\1$VERSION/" version.txt -sed -E -i "s/(FLATCAR_SDK_VERSION=)(.*)/\1$SDK_VERSION/" version.txt - -echo "Removing old build-*.xml files" -git rm ./build-*.xml || echo "Warning: Could not delete old files" - -FILENAME="build-$(echo "$VERSION" | cut -d '.' -f 1).xml" - -export SCRIPTS_REF OVERLAY_REF PORTAGE_REF DEFAULT_REF -cat "$SCRIPTFOLDER/manifest-template.xml.envsubst" | envsubst '$SCRIPTS_REF $OVERLAY_REF $PORTAGE_REF $DEFAULT_REF' > "$FILENAME" -# Note: appc-acbuild, appc-spec, rkt, and systemd always stay at refs/heads/master because they do not have flatcar-master or build branches - -ln -fs "$FILENAME" default.xml -cp "$FILENAME" release.xml - -MAJOR="${VERSION%%.*}" - -MAINT="flatcar" -if [ "$CHANNEL" = lts ] && [ "$MAJOR" = "2605" ]; then - MAINT="flatcar-lts" -fi - -echo "Creating maintenance.xml for $MAINT-$MAJOR branches" - -SCRIPTS_REF="refs/heads/$MAINT-$MAJOR" -OVERLAY_REF="refs/heads/$MAINT-$MAJOR" -PORTAGE_REF="refs/heads/$MAINT-$MAJOR" - -export SCRIPTS_REF OVERLAY_REF PORTAGE_REF DEFAULT_REF -cat "$SCRIPTFOLDER/manifest-template.xml.envsubst" | envsubst '$SCRIPTS_REF $OVERLAY_REF $PORTAGE_REF $DEFAULT_REF' > maintenance.xml - -echo "Adding changed files" -git add "$FILENAME" release.xml default.xml version.txt maintenance.xml -echo "Committing manifest" -git commit -m "build $VERSION" --author 'Flatcar Buildbot ' -echo "Pushing branch" - -git push --force-with-lease --set-upstream origin "$BRANCHNAME" -TAG="v$VERSION" -echo "Deleting tag $TAG if it exists" -git tag -d "$TAG" || echo "No local tags deleted" -git push --delete origin "$TAG" || echo "No remote tags deleted" -echo "Tagging commit" -git tag -s "$TAG" -m "$TAG" -echo "Pushing tag" -git push origin "$TAG" -echo "Done" diff --git a/devstats b/devstats index c50eb1e..7c3ed4e 100755 --- a/devstats +++ b/devstats @@ -32,7 +32,7 @@ fi # Excluded: flatcar/ignition flatcar/afterburn flatcar/ign-converter flatcar/grub # (because they are upstream projects) -REPOS=(flatcar/scripts flatcar/coreos-overlay flatcar/portage-stable flatcar/flatcar-website +REPOS=(flatcar/scripts flatcar/flatcar-website flatcar/init flatcar/flatcar-linux-update-operator flatcar/flatcar-build-scripts flatcar/mantle flatcar/update-ssh-keys flatcar/container-linux-config-transpiler flatcar/locksmith flatcar/bootengine flatcar/baselayout diff --git a/flatcar-overlay-diff.py b/flatcar-overlay-diff.py new file mode 100755 index 0000000..c025a53 --- /dev/null +++ b/flatcar-overlay-diff.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 + +from sh import which, grep, cut, mktemp, ErrorReturnCode # Requires "sh": sudo dnf install python3-sh +from sh.contrib import git +# Read docs here: http://amoffat.github.io/sh/index.html + +import argparse +import os +from pathlib import Path + +parser = argparse.ArgumentParser(description="Compare two flatcar-overlay branches including " + "dereferened EGIT_COMMIT branches of " + "repositories located in flatcar-overlay/../.") + +parser.add_argument("THEIRS", type=str, help="Reference/branch to compare to") +parser.add_argument("--ours", type=str, help="Our reference (defaults to \"HEAD\")") +parser.add_argument("--flatcar-overlay", "--coreos-overlay", type=str, help="Path to flatcar-overlay repository (defaults to \".\")") +parser.add_argument("--no-color", dest="no_color", action="store_true", help="Don't pipe diff through colordiff") +parser.add_argument("--no-commits", dest="no_commits", action="store_true", help="Don't log which commits are missing") +parser.add_argument("--no-diffs", dest="no_diffs", action="store_true", help="Don't show diffs") +parser.add_argument("--no-prs", dest="no_prs", action="store_true", help="Don't show PR links for merge commits " + "(ignoring cherry picks without a merge commit as cherry-pick-for can do)") +parser.add_argument("--diff-style", choices=["standard", "word-diff", "icdiff", "colordiff", "delta"], + help="Instead of standard git diff, use either git --word-diff, git icdiff, or pipe the existing diff through colordiff or delta") +parser.set_defaults(ours="HEAD", flatcar_overlay=".", no_color=False, no_commits=False, no_diffs=False) +args = parser.parse_args() + +base_folder = str(Path(args.flatcar_overlay + "/../").resolve()) + "/" + +if args.diff_style == "colordiff": + if not which("colordiff"): + raise Exception("colordiff not installed, try to run: sudo dnf install colordiff") + from sh import colordiff +elif args.diff_style == "delta": + if not which("delta"): + raise Exception("delta not installed, try to run: cargo install --git https://github.com/dandavison/delta") + from sh import delta + colordiff = delta.bake("--theme=none", "--color-only", "--paging=never") + +warnings = [] + +repo_map = {"coreos-init": "init", "cros-devutils": "flatcar-dev-util", "gmerge": "flatcar-dev-util", + "fero-client": "fero", "actool": "spec"} + + +def commits_to_pick(src="src", dst="dst"): + # Shows which commits in src can be picked from src to dst. + # The output is in the format of git log dst..src but excluding + # changes to .github and commits that are cherry-picked already. + # The function is equivalent to these two commands but with error checking: + # $ git cherry dst src | grep ^+ | cut -d " " -f 2 > tmp_outfile + # $ git log --no-merges --cherry-pick --format=%H dst..src -- . :!.github | grep -F -x -f tmp_outfile | xargs git show -s + # In our case --cherry-pick doesn't actually filter out we want to filter out + # (maybe a Flatcar branch strangeness), so we need to postprocess with git cherry. + tmp_outfile = str(mktemp("-u")).split("\n")[0] + commits_src_has_with_cherry = git.log("--no-merges", "--cherry-pick", # set --cherry-pick just in case it will filter out something already + "--format=%H", dst + ".." + src , "--", ".", ":!.github") + _commits_src_has_without_cherry = cut(grep(git.cherry(dst, src, _bg=False), + "^+", _ok_code=[1, 0]), + "-d", " ", "-f", "2", _out=tmp_outfile) + commits_src_has_filtered = str(grep(commits_src_has_with_cherry, + "-F", "-x", "-f", tmp_outfile, + _ok_code=[1, 0])).strip().split("\n") + commits_src_has_filtered = [commit for commit in commits_src_has_filtered if commit != ""] + if len(commits_src_has_filtered) > 0: + git_log = git.show("-s", *commits_src_has_filtered) + else: + git_log = "" + os.remove(tmp_outfile) + return git_log + +def pull_requests_for_merge_commits(src="src", dst="dst", repo="repo"): + # Prints links for GitHub PRs from merge commits. + # It uses merge commits which means that we don't find the PRs for cherry picks unless they have a merge commit + # as the cherry-pick-for script can do. + # The git log format is "subject#body" being "Merge pull request #NUMBER from BRANCH#PR_TITLE". + merge_commits = str(git.log("--merges", "--format=%s#%b", dst + ".." + src)) + filtered = [line for line in merge_commits.split("\n") if "Merge pull request" in line and line.count("#") >= 2] + # Won't panic because we ensured above that two # characters exist + pr_and_titles = [(line.split("#")[1].split(" ")[0], "#".join(line.split("#")[2:])) for line in filtered] + # TODO: find the correct upstream GitHub organization (from branch?) if it isn't kinvolk but systemd or coreos + # Ignores PRs that tell that they only change the .github folder by having a title starting with ".github" + links = [title + ": https://github.com/kinvolk/" + repo + "/pull/" + pr for (pr, title) in pr_and_titles if not title.startswith(".github")] + return "\n".join(links) + +def display_difference(from_theirs, to_ours, name, recurse=False): + # That means, show what "our" branch adds to "their" branch + diff_args = [from_theirs + ".." + to_ours, "--", ".", ":!.github"] + diff = git.diff(*diff_args, _bg=False, _decode_errors="replace") + desc_start = "↓" * 25 + desc_end = "↑" * 25 + desc = "Diff for " + name + if not args.no_diffs: + print(desc_start, desc, desc_start + "\n") + if args.diff_style == "icdiff": + print(git.difftool("-y", "-x", "icdiff --is-git-diff --cols=160", *diff_args, _bg=False, _decode_errors="replace")) + elif args.diff_style == "word-diff": + print(git.diff("--word-diff", "--no-color" if args.no_color else "--color", *diff_args, _bg=False, _decode_errors="replace")) + elif args.no_color: + print(diff) + elif args.diff_style == "colordiff" or args.diff_style == "delta": + print(colordiff(diff, _decode_errors="replace")) + else: + print(git.diff("--color", *diff_args, _bg=False, _decode_errors="replace")) + print("\n" + desc_end, desc, desc_end + "\n") + if not args.no_commits: + # Branch "our" is the checked out branch and "git diff" shows what "our" branch has that "their" branch doesn't. + # Converting the diff to "our" commits to pick for "their" branch) means setting "src" as "our". + commits_we_have = commits_to_pick(src=to_ours, dst=from_theirs) + commits_they_have = commits_to_pick(src=from_theirs, dst=to_ours) + desc = "Commits for " + name + " in our " + to_ours + " but not in their " + from_theirs + print(desc_start, desc, desc_start + "\n") + print(commits_we_have) + print("\n" + desc_end, desc, desc_end + "\n") + desc = "Commits for " + name + " in their " + from_theirs + " but not in our " + to_ours + print(desc_start, desc, desc_start + "\n") + print(commits_they_have) + print("\n" + desc_end, desc, desc_end + "\n") + if not args.no_prs: + prs_we_have = pull_requests_for_merge_commits(src=to_ours, dst=from_theirs, repo=name) + prs_they_have = pull_requests_for_merge_commits(src=from_theirs, dst=to_ours, repo=name) + desc = "PRs (from merge commits) for " + name + " in our " + to_ours + " but not in their " + from_theirs + print(desc_start, desc, desc_start + "\n") + print(prs_we_have) + print("\n" + desc_end, desc, desc_end + "\n") + desc = "PRs (from merge commits) for " + name + " in their " + from_theirs + " but not in our " + to_ours + print(desc_start, desc, desc_start + "\n") + print(prs_they_have) + print("\n" + desc_end, desc, desc_end + "\n") + if recurse: + theirs = "" + ours = "" + repo = "" + for line in diff.splitlines(): + if line.startswith("diff --git ") and line.endswith("ebuild"): + if theirs != "" or ours != "": + warnings.append("Error: Unexpected variable content (theirs: [expected empty]: " + theirs + ", ours [expected empty]: " + ours + ") for " + repo) + theirs = "" + ours = "" + repo = "-".join(line.split("/")[-1].split("-")[:-1]) # Get "some-name" for ".../some-name-9999.ebuild" + if repo in repo_map: + repo = repo_map[repo] + # @TODO: Add DOCKER_GITCOMMIT, COMMIT_ID, CONTAINERD_COMMIT + if "EGIT_COMMIT=" in line: + if repo == "": + raise Exception("No repo seen for: " + line) + is_theirs = line.startswith("-") + is_ours = line.startswith("+") + if not is_theirs and not is_ours: + raise Exception("Unexpected line:" + line) + # Checks that "- ..." is followed by "+ ..." + if is_theirs: + if theirs != "" or ours != "": + warnings.append("Error: Unexpected variable content, expected empty (theirs: " + theirs + ", ours: " + ours + ") for " + repo + " in: " + line) + theirs = "" + ours = "" + repo = "" + continue + theirs = line.split("\"")[1] + if is_ours: + if theirs == "" or ours != "": + warnings.append("Error: Unexpected variable content (theirs [expected not empty]: " + theirs + ", ours [expected empty]: " + ours + ") for " + repo + " in: " + line) + theirs = "" + ours = "" + repo = "" + continue + ours = line.split("\"")[1] + if theirs != "" and ours != "": + os.chdir(base_folder) + try: + os.chdir(base_folder + repo) + except FileNotFoundError: + print("Failed to enter repo directory for \"" + repo + "\", trying to clone it") + git.clone("git@github.com:kinvolk/" + repo + ".git") + os.chdir(repo) + try: + git.fetch("github") + except ErrorReturnCode: + print("Tried to fetch from github without success, trying to fetch the default remote.") + git.fetch() + print(desc_start, "Difference for", repo, desc_start + "\n") + display_difference(theirs, ours, repo) + print("\n" + desc_end, "Difference for", repo, desc_end + "\n") + repo = "" + theirs = "" + ours = "" + + +os.chdir(args.flatcar_overlay) +display_difference(args.THEIRS, args.ours, os.path.basename(os.path.abspath(".")), recurse=True) +if warnings: + print("Encountered some errors when trying to compare recursively, probably due to deleted files:") + print("\n".join(warnings)) + print() +print("Done. Displayed all differences.") diff --git a/lib/common.sh b/lib/common.sh index 8d98848..fd2b5ba 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -9,7 +9,6 @@ export FLATCAR_REPOS=( "bootengine" "chromite" "coreos-cloudinit" - "coreos-overlay" "dev-util" "docker" "efunctions" @@ -23,7 +22,6 @@ export FLATCAR_REPOS=( "mantle" "mayday" "nss-altfiles" - "portage-stable" "rkt" "scripts" "sdnotify-proxy" @@ -36,8 +34,3 @@ export FLATCAR_REPOS=( "update_engine" "updateservicectl" ) - -export FLATCAR_REPOS_MANIFEST=( - "manifest" - "manifest-builds" -) diff --git a/manifest-template.xml.envsubst b/manifest-template.xml.envsubst deleted file mode 100644 index 6a75a86..0000000 --- a/manifest-template.xml.envsubst +++ /dev/null @@ -1,37 +0,0 @@ - - - Your sources have been sync'd successfully. Note that only the revisions for the coreos-overlay, scripts, and portage-stable repositories are relevant. Revisions for the other repositories are specified in the ebuild files. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/mirror-repos-branch b/mirror-repos-branch index a315319..36e67f6 100755 --- a/mirror-repos-branch +++ b/mirror-repos-branch @@ -30,7 +30,7 @@ fi . ./lib/common.sh if [ "${ALL_REPOS}" = "0" ]; then - FLATCAR_REPOS=("coreos-overlay" "portage-stable" "scripts") + FLATCAR_REPOS=("scripts") fi REPOS_DIR=$(mktemp -d "${PWD}/.mirror-repos.XXXXXXXXXX") @@ -41,12 +41,6 @@ trap '{ export EXT="$?"; rm -rf "${REPOS_DIR}" && exit "${EXT}"; }' EXIT cd "${REPOS_DIR}" -# Check if we are in the monorepo case -[ ! -d scripts ] && git clone --recurse-submodules "git@github.com:flatcar/scripts" -if [ "$(git -C scripts show "origin/${SRC_BUILD_BRANCH}":.gitmodules 2>/dev/null)" = "" ]; then - FLATCAR_REPOS=("scripts") -fi - for repo in "${FLATCAR_REPOS[@]}"; do # Note: systemd repo has neither flatcar-build-* branches nor flatcar-master # branch. So we should skip systemd completely. diff --git a/show-changes b/show-changes index 49ead14..b18f66a 100755 --- a/show-changes +++ b/show-changes @@ -7,14 +7,12 @@ SCRIPTFOLDER="$(dirname "$(readlink -f "$0")")" if [ $# -lt 1 ] || [ "$1" = "-h" ] || [ "$1" = "--help" ]; then echo "Usage: $0 OLD [NEW]" echo "Shows the changes between the git references by assembling the changelog/ folder entries" - echo "By default the script assumes that it is being run in the folder that contains the coreos-overlay, portage-stable, and scripts repository folders. Set SCRIPTS_REPO, COREOS_OVERLAY_REPO and PORTAGE_REPO to contain the paths to the respective repos if the default assumption is wrong." + echo "By default the script assumes that it is being run in the folder that contains the scripts repository folder. Set SCRIPTS_REPO to contain the path to the repo if the default assumption is wrong." echo "The NEW reference can be omitted and will then default to HEAD." exit 1 fi : "${SCRIPTS_REPO:=scripts}" -: "${COREOS_OVERLAY_REPO:=coreos-overlay}" -: "${PORTAGE_STABLE_REPO:=portage-stable}" OLD="$1" NEW="${2-HEAD}" @@ -32,7 +30,7 @@ fi echo "_Changes since **${OLD_FMT}**_" if [[ ${FETCH} = 1 ]]; then - for repo in coreos-overlay portage-stable scripts; do + for repo in scripts; do var_name=${repo//-/_} var_name="${var_name^^}_REPO" if [[ -d ${!var_name} ]]; then @@ -61,44 +59,15 @@ for section in security bugfixes changes updates; do exit 1 esac echo - for repo in coreos-overlay portage-stable scripts; do + for repo in scripts; do var_name=${repo//-/_} var_name="${var_name^^}_REPO" OLDREF="${OLD}" NEWREF="${NEW}" - OLDREPOPATH="${!var_name}" - NEWREPOPATH="${!var_name}" - OLDPREPEND="" - NEWPREPEND="" - if [ "${repo}" != "scripts" ]; then - # Check if we are in the monorepo case where we can use the scripts ref or not - if [ "$(git -C "${SCRIPTS_REPO}" show "${OLD}":.gitmodules 2>/dev/null)" != "" ]; then - # Old version is not a monorepo but has submodules. - # Find the pinned submodule refs because there may be no release tags inside the submodules - # Pipe to awk instead of using --object-only for git 2.35 support - OLDREF=$(git -C "${SCRIPTS_REPO}" ls-tree "${OLD}" "sdk_container/src/third_party/${repo}" | awk '{print $3 }') - # We can't assume anymore that the submodule repo is available under scripts/ - if [ ! -d "${OLDREPOPATH}" ]; then - git clone "git@github.com:flatcar/${repo}.git" "${OLDREPOPATH}" - fi - else - OLDPREPEND="sdk_container/src/third_party/${repo}/" - OLDREPOPATH="${SCRIPTS_REPO}" - fi - if [ "$(git -C "${SCRIPTS_REPO}" show "${NEW}":.gitmodules 2>/dev/null)" != "" ]; then - # New version is not a monorepo but has submodules. - NEWREF=$(git -C "${SCRIPTS_REPO}" ls-tree "${NEW}" "sdk_container/src/third_party/${repo}" | awk '{print $3 }') - if [ ! -d "${NEWREPOPATH}" ]; then - git clone "git@github.com:flatcar/${repo}.git" "${NEWREPOPATH}" - fi - else - NEWPREPEND="sdk_container/src/third_party/${repo}/" - NEWREPOPATH="${SCRIPTS_REPO}" - fi - fi - if [ "${section}" = "security" ] && [ "${repo}" = "coreos-overlay" ]; then - FROM_KERNEL=$(git -C "${OLDREPOPATH}" show "${OLDREF}":"${OLDPREPEND}"sys-kernel/coreos-kernel/ | grep -m 1 'coreos-kernel-.*\.ebuild' | cut -d - -f 3 | cut -d . -f 1-3) - TO_KERNEL=$(git -C "${NEWREPOPATH}" show "${NEWREF}":"${NEWPREPEND}"sys-kernel/coreos-kernel/ | grep -m 1 'coreos-kernel-.*\.ebuild' | cut -d - -f 3 | cut -d . -f 1-3) + REPOPATH="${!var_name}" + if [ "${section}" = "security" ] && [ "${repo}" = "scripts" ]; then + FROM_KERNEL=$({ git -C "${REPOPATH}" show "${OLDREF}":repos/flatcar-overlay/sys-kernel/coreos-kernel/ 2>/dev/null || git -C "${REPOPATH}" show "${OLDREF}":sdk_container/src/third_party/coreos-overlay/sys-kernel/coreos-kernel/; } | grep -m 1 'coreos-kernel-.*\.ebuild' | cut -d - -f 3 | cut -d . -f 1-3) + TO_KERNEL=$({ git -C "${REPOPATH}" show "${NEWREF}":repos/flatcar-overlay/sys-kernel/coreos-kernel/ 2>/dev/null || git -C "${REPOPATH}" show "${NEWREF}":sdk_container/src/third_party/coreos-overlay/sys-kernel/coreos-kernel/; } | grep -m 1 'coreos-kernel-.*\.ebuild' | cut -d - -f 3 | cut -d . -f 1-3) if [ "${FROM_KERNEL}" != "${TO_KERNEL}" ]; then KERNEL_ENTRIES=$("${SCRIPTFOLDER}"/show-fixed-kernel-cves.py -f "${FROM_KERNEL}" -t "${TO_KERNEL}") if [ "${KERNEL_ENTRIES}" != "" ]; then @@ -107,18 +76,7 @@ for section in security bugfixes changes updates; do fi fi - # The assumption is that the old ref is really older, so we can assume that old would have submodules while new doesn't have them anymore - if [ "${OLDREPOPATH}" != "${NEWREPOPATH}" ] && [ "${NEWREPOPATH}" = "${SCRIPTS_REPO}" ]; then - # One patch before the ./checkout helper disappeared we still had submodules - LAST_SUBMOD_SCRIPTS_REF="$(git -C "${SCRIPTS_REPO}" rev-list -n 1 "${NEWREF}" -- checkout)~1" - LAST_SUBMOD_REF=$(git -C "${SCRIPTS_REPO}" ls-tree "${LAST_SUBMOD_SCRIPTS_REF}" "sdk_container/src/third_party/${repo}" | awk '{print $3 }') - # The patch that removed the submodule overrides README has the merge history - FIRST_MONO_REF=$(git -C "${SCRIPTS_REPO}" rev-list -n 1 "${NEWREF}" -- sdk_container/git-override/README.md) - git -C "${OLDREPOPATH}" difftool --no-prompt --extcmd='sh -c "cat \"$REMOTE\"" --' "${OLDREF}..${LAST_SUBMOD_REF}" -- "${OLDPREPEND}changelog/${section}/" | sort || { echo "Error: git difftool failed" ; exit 1 ; } - git -C "${NEWREPOPATH}" difftool --no-prompt --extcmd='sh -c "cat \"$REMOTE\"" --' "${FIRST_MONO_REF}..${NEWREF}" -- "${NEWPREPEND}changelog/${section}/" | sort || { echo "Error: git difftool failed" ; exit 1 ; } - else - git -C "${NEWREPOPATH}" difftool --no-prompt --extcmd='sh -c "cat \"$REMOTE\"" --' "${OLDREF}..${NEWREF}" -- "${NEWPREPEND}changelog/${section}/" | sort || { echo "Error: git difftool failed" ; exit 1 ; } - fi + git -C "${REPOPATH}" difftool --no-prompt --extcmd='sh -c "cat \"$REMOTE\"" --' "${OLDREF}..${NEWREF}" -- "changelog/${section}/" | sort || { echo "Error: git difftool failed" ; exit 1 ; } # The -x 'sh -c "cat \"$REMOTE\"" --' command assumes that new changes have their own changelog files, # and thus ignores the LOCAL file (which is the empty /dev/null) and prints out the REMOTE completly. # If an existing file got changed, we assume that this is just a correction for the old change but diff --git a/sync-with-gentoo b/sync-with-gentoo index 1a13851..8b334b2 100755 --- a/sync-with-gentoo +++ b/sync-with-gentoo @@ -1,7 +1,7 @@ #!/bin/bash # Used for syncing with gentoo. Needs to be called from the -# toplevel-directory of portage-stable. Expects the actual gentoo repo +# toplevel-directory of gentoo-subset. Expects the actual gentoo repo # to be either in ../gentoo or ../../gentoo. # # Example invocations: diff --git a/tag-release b/tag-release index 3d2296e..1fb469a 100755 --- a/tag-release +++ b/tag-release @@ -4,16 +4,15 @@ SCRIPTFOLDER="$(dirname "$(readlink -f "$0")")" if [ "${SDK_VERSION}" = "" ] || [ "$VERSION" = "" ] || [ "$CHANNEL" = "" ] || [ "$1" = "-h" ] || [ "$1" = "--help" ]; then echo "$0:" - echo "This script will create and push release tags in the repositories scripts, coreos-overlay, and portage-stable" - echo "checked out in $SCRIPTFOLDER/../(scripts|coreos-overlay|portage-stable). The repositories will be cloned from" - echo "github.com/flatcar/(scripts|coreos-overlay|portage-stable).git if they do not exist (origin in an existing" - echo "repository must point there or pushing fails or does some unwanted action)." + echo "This script will create and push release tags in the scripts repository checked out in" + echo "$SCRIPTFOLDER/../scripts. The repository will be cloned from github.com/flatcar/scripts.git" + echo "if it does not exist (origin in an existing repository must point there or pushing fails or does some unwanted action)." echo "Set VERSION, SDK_VERSION, and CHANNEL as environment variables, e.g., VERSION=2345.3.0 SDK_VERSION=2345.0.0 CHANNEL=stable $0" echo echo "It creates tags in the form CHANNEL-VERSION for each of the three repositories. By default it fetches origin and" echo "creates a tag from origin/flatcar-MAJOR for each of the three repositories, with MAJOR being the first part of" echo "VERSION when splitting by dots." - echo "Set the environment variables SCRIPTS_REF, OVERLAY_REF, PORTAGE_REF to specify a specific commit for the tag." + echo "Set the environment variable SCRIPTS_REF to specify a specific commit for the tag." exit 1 fi @@ -21,64 +20,42 @@ set -euo pipefail MAJOR="${VERSION%%.*}" -MAINT="flatcar" -if [ "$CHANNEL" = lts ] && [ "$MAJOR" = "2605" ]; then - MAINT="flatcar-lts" -fi - -SCRIPTS_REF="${SCRIPTS_REF-origin/$MAINT-$MAJOR}" -OVERLAY_REF="${OVERLAY_REF-origin/$MAINT-$MAJOR}" -PORTAGE_REF="${PORTAGE_REF-origin/$MAINT-$MAJOR}" +SCRIPTS_REF="${SCRIPTS_REF-origin/flatcar-$MAJOR}" echo "Running with CHANNEL=$CHANNEL VERSION=$VERSION MAJOR=$MAJOR" -echo "SCRIPTS_REF=$SCRIPTS_REF OVERLAY_REF=$OVERLAY_REF PORTAGE_REF=$PORTAGE_REF" - -REPOS="coreos-overlay portage-stable scripts" -# scripts should be last as it binds the other two together as submodules, if they exist - -# Check if we are in the monorepo, in this case only work on "scripts" -cd "$SCRIPTFOLDER/.." -if [ ! -d "scripts" ]; then - git clone --recurse-submodules 'git@github.com:flatcar/scripts.git' -fi -git -C scripts fetch origin -if [ "$(git -C scripts show "origin/$MAINT-$MAJOR":.gitmodules 2>/dev/null)" = "" ]; then - REPOS="scripts" -fi +echo "SCRIPTS_REF=$SCRIPTS_REF" +REPOS="scripts" for REPO in ${REPOS}; do echo "Preparing ${REPO}" cd "$SCRIPTFOLDER/.." if [ ! -d "${REPO}" ]; then - git clone --recurse-submodules "git@github.com:flatcar/${REPO}.git" + git clone "git@github.com:flatcar/${REPO}.git" fi cd "${REPO}" git fetch origin TAG="${CHANNEL}-${VERSION}" [ "${REPO}" = "scripts" ] && REF="${SCRIPTS_REF}" - [ "${REPO}" = "coreos-overlay" ] && REF="${OVERLAY_REF}" - [ "${REPO}" = "portage-stable" ] && REF="${PORTAGE_REF}" echo "Deleting tag ${TAG} if it exists in ${REPO}" git tag -d "$TAG" || echo "No local tags deleted" git push --delete origin "$TAG" || echo "No remote tags deleted" - # Check if we have to update the submodules while tagging if [ "${REPO}" = "scripts" ]; then - if [ "${REF}" != "origin/$MAINT-$MAJOR" ]; then - echo "Error: can't find the scripts branch to push the updated submodule to, you can't overwrite SCRIPTS_REF anymore" + if [ "${REF}" != "origin/flatcar-$MAJOR" ]; then + echo "Error: can't find the scripts branch, you can't overwrite SCRIPTS_REF anymore" exit 1 fi - echo "Checking out scripts branch $MAINT-$MAJOR to update submodules and create version file" - git checkout --force --recurse-submodules -B "$MAINT-$MAJOR" "origin/$MAINT-$MAJOR" || { echo "Error: could not checkout the right branch in your 'scripts' repo" ; exit 1 ; } + echo "Checking out scripts branch flatcar-$MAJOR to create version file" + git checkout --force -B "flatcar-$MAJOR" "origin/flatcar-$MAJOR" || { echo "Error: could not checkout the right branch in your 'scripts' repo" ; exit 1 ; } if [ "$(git status --porcelain --untracked-files=no)" != "" ]; then echo "Error: uncommitted changes found which will be lost" exit 1 fi - git pull --recurse-submodules || { echo "Error: could not pull the branch in your 'scripts' repo" ; exit 1 ; } - if [ "$(git log HEAD.."origin/$MAINT-$MAJOR")" != "" ] || ! git diff --quiet "origin/$MAINT-$MAJOR" ; then + git pull || { echo "Error: could not pull the branch in your 'scripts' repo" ; exit 1 ; } + if [ "$(git log HEAD.."origin/flatcar-$MAJOR")" != "" ] || ! git diff --quiet "origin/flatcar-$MAJOR" ; then echo "Error: local changes in your 'scripts' repo" exit 1 fi @@ -86,13 +63,8 @@ for REPO in ${REPOS}; do ( source sdk_lib/sdk_container_common.sh source ci-automation/ci_automation_common.sh - if [ "$(git show "origin/$MAINT-$MAJOR":.gitmodules 2>/dev/null)" != "" ]; then - cd "sdk_container/src/third_party/coreos-overlay"; git fetch --all --tags --force; cd - - cd "sdk_container/src/third_party/portage-stable"; git fetch --all --tags --force; cd - - update_submodules "${OVERLAY_REF}" "${PORTAGE_REF}" - fi create_versionfile "${SDK_VERSION}" "${VERSION}" - SIGN=1 update_and_push_version "${TAG}" "$MAINT-$MAJOR" + SIGN=1 update_and_push_version "${TAG}" "flatcar-$MAJOR" ) else # Tag the other repos