forked from os-autoinst/scripts
-
Notifications
You must be signed in to change notification settings - Fork 0
/
_common
192 lines (167 loc) · 6.68 KB
/
_common
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
#!/bin/bash -e
# Common shell script snippets to be used when interacting with openQA
# instances, for example over openqa-cli.
enable_force_result=${enable_force_result:-false}
client_call=()
client_args=()
markdown_cmd=$(command -v Markdown.pl) || markdown_cmd=$(command -v markdown)
warn() { echo "$@" >&2; }
error-handler() {
local line=$1
# shellcheck disable=SC2207
local c=($(caller))
echo "${c[1]}: ERROR: line $line" >&2
}
# From openqa-cli JSON output filter and return the id/ids of jobs,
# for example from a query to the 'jobs get' route or the result string of a
# 'jobs post' or 'isos post'
job_ids() {
jq -r '.ids[]' "$@"
}
# Wrapper around openqa-cli that outputs the api arguments in case of
# an error, and the calling command and line
runcli() {
local args=("$@")
local rc output
set +e
output=$("${args[@]}" 2>&1)
rc=$?
set -e
[[ "$rc" == 0 ]] && echo "$output" && return
warn "openqa-cli ($(caller)): (${args[*]}) rc: $rc >>>$output<<<"
return $rc
}
# Wrapper around jq that outputs the first lines of JSON in case
# jq has a problem with it, and the calling command and line
runjq() {
local rc output
local jq_output_limit="${jq_output_limit:-15}"
input=$(</dev/stdin)
set +e
output=$(echo "$input" | jq "$@" 2>&1)
rc=$?
set -e
[[ "$rc" == 0 ]] && echo "$output" && return
output=$(echo "$output" | head -"$jq_output_limit")
echo "jq ($(caller)): $output (rc: $rc Input: >>>$input<<<)" >&2
return $rc
}
# Wrapper around curl that reports the HTTP status if it is not 200, and the
# calling command and line
runcurl() {
local rc status_code body response
local verbose="${verbose:-"false"}"
$verbose && echo "[debug] curl: Fetching ($*)" >&2
set +e
response=$(curl -w "\n%{http_code}\n" "$@" 2>&1)
rc=$?
set -e
[[ "$rc" != 0 ]] && echo "curl ($(caller)): Error fetching ($*): $response" >&2 && return 1
status_code=$(echo "$response" | tail -1)
[[ "$status_code" != 200 ]] && echo "curl ($(caller)): Error fetching url ($*): Got Status $status_code" >&2 && return 1
# remove last line
body=$(echo "$response" | tac | tail -n+2 | tac)
echo "$body"
}
comment_on_job() {
local id=$1 comment=$2 force_result=${3:-''}
if $enable_force_result && [[ -n $force_result ]]; then
comment="label:force_result:$force_result:$comment
$comment"
fi
"${client_call[@]}" -X POST jobs/"$id"/comments text="$comment"
}
search_log() {
local id=$1 search_term=$2 out=$3 grep_timeout=${4:-5}
local rc=0 grep_output
local grep_opts="${grep_opts:-"-qPzo"}"
# shellcheck disable=SC2086
grep_output=$(timeout "$grep_timeout" grep $grep_opts "$search_term" "$out" 2>&1) || rc=$?
if [[ "$rc" == 1 ]]; then
return 1
elif [[ "$rc" == 124 ]]; then
warn "grep was killed, possibly timed out: cmd=>grep $grep_opts '$search_term' '$out'< output='$grep_output'"
return $rc
elif [[ "$rc" != 0 ]]; then
# unexpected error, e.g. "exceeded PCRE's backtracking limit"
warn "grep failed: cmd=>grep $grep_opts '$search_term' '$out'< output='$grep_output'"
return $rc
fi
}
label_on_issue() {
local id=$1 search_term=$2 label=$3 restart=${4:-''} force_result=${5:-''}
local out=${out:?}
search_log "$id" "$search_term" "$out" || return
comment_on_job "$id" "$label" "$force_result"
if [ "$restart" = "1" ]; then
"${client_call[@]}" -X POST jobs/"$id"/restart
fi
}
snip_start=" # --- 8< ---"$'\n'
snip_end=" # --- >8 ---"
handle_unknown() {
local testurl=$1 out=$2 reason=$3 group_id=$4 email_unreviewed=$5 from_email=$6 notification_address=${7:-} job_data=${8:-} dry_run=${9:-}
local group_data group_name group_description group_mailto header email excerpt=$snip_start
local job_name job_result job_info
to_review+=("$testurl ${reason:0:50}")
header="[$testurl]($testurl): Unknown issue, to be reviewed"$'\n'"-> [autoinst-log.txt]($testurl/file/autoinst-log.txt)"$'\n'
excerpt="Likely the error is within this log excerpt, last lines before shutdown:"$'\n'$'\n'"$snip_start"
# Look for different termination points with likely context
excerpt+=$( (grep -A 12 'Backend process died, backend errors are reported below in the following lines' "$out" || grep -B 10 'sending magic and exit' "$out" || grep -B 5 'killing command server.*because test execution ended through exception' "$out" || grep -B 5 'EXIT 1' "$out" || grep -B 10 '\(Result: died\|isotovideo failed\)' "$out" || echo '(No log excerpt found)') | sed 's/^/ # /' | head -n -1 | sed 's/\x1b\[[0-9;]*m//g')
excerpt+=$'\n'"$snip_end"
echo "$header"
echo "$excerpt"
if "$email_unreviewed" && [[ "$group_id" != 'null' ]]; then
group_data=$(openqa-cli "${client_args[@]}" "job_groups/$group_id")
group_description=$(echo "$group_data" | runjq -r '.[0].description' ) || true
group_mailto=$(echo "$group_description" | perl -n -wE'm/.*MAILTO: (\S+).*/ and say $1' | head -1)
group_mailto=${group_mailto:-$notification_address}
# Avoid sending notifications for jobs that were restarted
clone_id="$(echo "$job_data" | runjq -r '.job.clone_id')"
if [[ -n "$group_mailto" && "$clone_id" == 'null' ]]; then
group_name=$(echo "$group_data" | runjq -r '.[0].name')
job_name=$(echo "$job_data" | runjq -r '.job.name')
job_result=$(echo "$job_data" | runjq -r '.job.result')
job_info="* Name: $job_name
* Result: $job_result
* Reason: $reason"
email="[$job_name]($testurl)"$'\n'"$header"$'\n'"$job_info"$'\n'$'\n'"$excerpt"
subject="Unreviewed issue (Group $group_id $group_name)"
email=$(multipart-from-markdown "$email" "$group_mailto" "openqa-label-known-issues <$from_email>" "$subject")
send-email "$group_mailto" "$email"
fi
fi
}
send-email() {
local mailto=$1 email=$2
if [[ $dry_run == 1 ]]; then
echo "Would send email to '$mailto':"$'\n'"$email"
return
fi
echo "$email" | /usr/sbin/sendmail -t "$mailto"
}
multipart-from-markdown() {
local plain=$1 to_email=$2 from=$3 subject=$4
local email md body header boundary
md=$(echo "$plain" | $markdown_cmd)
boundary="_000_DB6PR0401MB2565_"
header='Content-Type: multipart/alternative; boundary="'"$boundary"'"'
body="--$boundary"'
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
'"$plain
--$boundary"'
Content-Type: text/html; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
<html><body>
'"$md"'
</body></html>
'"--$boundary--"
email="To: $to_email
From: $from
Subject: $subject
$header
$body
"
echo "$email"
}